hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jia...@apache.org
Subject [05/11] hadoop git commit: YARN-6335. Port slider's groovy unit tests to yarn native services. Contributed by Billie Rinaldi
Date Thu, 20 Apr 2017 15:53:34 GMT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
new file mode 100644
index 0000000..7d8f5a7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
+import org.apache.slider.server.appmaster.model.mock.MockContainer;
+import org.apache.slider.server.appmaster.model.mock.MockNodeId;
+import org.apache.slider.server.appmaster.model.mock.MockPriority;
+import org.apache.slider.server.appmaster.model.mock.MockResource;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome;
+import org.apache.slider.server.appmaster.state.ContainerAllocationResults;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.OutstandingRequest;
+import org.apache.slider.server.appmaster.state.OutstandingRequestTracker;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test outstanding request tracker.
+ */
+public class TestRoleHistoryOutstandingRequestTracker extends
+    BaseMockAppStateTest {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestRoleHistoryOutstandingRequestTracker.class);
+
+  public static final String WORKERS_LABEL = "workers";
+  private NodeInstance host1 = new NodeInstance("host1", 3);
+  private NodeInstance host2 = new NodeInstance("host2", 3);
+  private MockResource resource = factory.newResource(48, 1);
+
+  private OutstandingRequestTracker tracker = new OutstandingRequestTracker();
+
+  public static final String WORKER = "worker";
+
+  @Override
+  public Application buildApplication() {
+    Application application = super.buildApplication();
+    Component component = new Component().name("worker").numberOfContainers(0L);
+    component.getConfiguration().setProperty(ResourceKeys.YARN_LABEL_EXPRESSION,
+        WORKERS_LABEL);
+    application.getComponents().add(component);
+    return application;
+  }
+
+  @Test
+  public void testAddRetrieveEntry() throws Throwable {
+    OutstandingRequest request = tracker.newRequest(host1, 0);
+    assertEquals(tracker.lookupPlacedRequest(0, "host1"), request);
+    assertEquals(tracker.removePlacedRequest(request), request);
+    assertNull(tracker.lookupPlacedRequest(0, "host1"));
+  }
+
+  @Test
+  public void testAddCompleteEntry() throws Throwable {
+    OutstandingRequest req1 = tracker.newRequest(host1, 0);
+    req1.buildContainerRequest(resource, getRole0Status(), 0);
+
+    tracker.newRequest(host2, 0).buildContainerRequest(resource,
+        getRole0Status(), 0);
+    tracker.newRequest(host1, 1).buildContainerRequest(resource,
+        getRole0Status(), 0);
+
+    ContainerAllocationResults allocation = tracker.onContainerAllocated(1,
+        "host1", null);
+    assertEquals(allocation.outcome, ContainerAllocationOutcome.Placed);
+    assertTrue(allocation.operations.get(0) instanceof CancelSingleRequest);
+
+    assertNull(tracker.lookupPlacedRequest(1, "host1"));
+    assertNotNull(tracker.lookupPlacedRequest(0, "host1"));
+  }
+
+  @Test
+  public void testResetOpenRequests() throws Throwable {
+    OutstandingRequest req1 = tracker.newRequest(null, 0);
+    assertFalse(req1.isLocated());
+    tracker.newRequest(host1, 0);
+    List<OutstandingRequest> openRequests = tracker.listOpenRequests();
+    assertEquals(1, openRequests.size());
+    tracker.resetOutstandingRequests(0);
+    assertTrue(tracker.listOpenRequests().isEmpty());
+    assertTrue(tracker.listPlacedRequests().isEmpty());
+  }
+
+  @Test
+  public void testRemoveOpenRequestUnissued() throws Throwable {
+    OutstandingRequest req1 = tracker.newRequest(null, 0);
+    req1.buildContainerRequest(resource, getRole0Status(), 0);
+    assertEquals(1, tracker.listOpenRequests().size());
+    MockContainer c1 = factory.newContainer(null, new MockPriority(0));
+    c1.setResource(resource);
+
+    ContainerAllocationResults allocation =
+        tracker.onContainerAllocated(0, "host1", c1);
+    ContainerAllocationOutcome outcome = allocation.outcome;
+    assertEquals(outcome, ContainerAllocationOutcome.Unallocated);
+    assertTrue(allocation.operations.isEmpty());
+    assertEquals(1, tracker.listOpenRequests().size());
+  }
+
+  @Test
+  public void testIssuedOpenRequest() throws Throwable {
+    OutstandingRequest req1 = tracker.newRequest(null, 0);
+    req1.buildContainerRequest(resource, getRole0Status(), 0);
+    assertEquals(1, tracker.listOpenRequests().size());
+
+    int pri = ContainerPriority.buildPriority(0, false);
+    assertTrue(pri > 0);
+    MockNodeId nodeId = factory.newNodeId("hostname-1");
+    MockContainer c1 = factory.newContainer(nodeId, new MockPriority(pri));
+
+    c1.setResource(resource);
+
+    ContainerRequest issued = req1.getIssuedRequest();
+    assertEquals(issued.getCapability(), resource);
+    assertEquals(issued.getPriority().getPriority(), c1.getPriority()
+        .getPriority());
+    assertTrue(req1.resourceRequirementsMatch(resource));
+
+    ContainerAllocationResults allocation =
+        tracker.onContainerAllocated(0, nodeId.getHost(), c1);
+    assertEquals(0, tracker.listOpenRequests().size());
+    assertTrue(allocation.operations.get(0) instanceof CancelSingleRequest);
+
+    assertEquals(allocation.outcome, ContainerAllocationOutcome.Open);
+    assertEquals(allocation.origin, req1);
+  }
+
+  @Test
+  public void testResetEntries() throws Throwable {
+    tracker.newRequest(host1, 0);
+    tracker.newRequest(host2, 0);
+    tracker.newRequest(host1, 1);
+    List<NodeInstance> canceled = tracker.resetOutstandingRequests(0);
+    assertEquals(2, canceled.size());
+    assertTrue(canceled.contains(host1));
+    assertTrue(canceled.contains(host2));
+    assertNotNull(tracker.lookupPlacedRequest(1, "host1"));
+    assertNull(tracker.lookupPlacedRequest(0, "host1"));
+    canceled = tracker.resetOutstandingRequests(0);
+    assertEquals(0, canceled.size());
+    assertEquals(1, tracker.resetOutstandingRequests(1).size());
+  }
+
+  @Test
+  public void testEscalation() throws Throwable {
+    // first request: default placement
+    assertEquals(getRole0Status().getPlacementPolicy(), PlacementPolicy
+        .DEFAULT);
+    Resource res0 = newResource(getRole0Status());
+    OutstandingRequest outstanding0 = tracker.newRequest(host1,
+        getRole0Status().getKey());
+    ContainerRequest initialRequest =
+        outstanding0.buildContainerRequest(res0, getRole0Status(), 0);
+    assertNotNull(outstanding0.getIssuedRequest());
+    assertTrue(outstanding0.isLocated());
+    assertFalse(outstanding0.isEscalated());
+    assertFalse(initialRequest.getRelaxLocality());
+    assertEquals(1, tracker.listPlacedRequests().size());
+
+    // second. This one doesn't get launched. This is to verify that the
+    // escalation process skips entries which are in the list but have not
+    // been issued, which can be a race condition between request issuance &
+    // escalation.
+    // (not one observed outside test authoring, but retained for completeness)
+    Resource res2 = newResource(getRole2Status());
+    OutstandingRequest outstanding2 = tracker.newRequest(host1,
+        getRole2Status().getKey());
+
+    // simulate some time escalation of role 1 MUST now be triggered
+    long interval = getRole0Status().getPlacementTimeoutSeconds() * 1000 + 500;
+    long now = interval;
+    final List<AbstractRMOperation> escalations = tracker
+        .escalateOutstandingRequests(now);
+
+    assertTrue(outstanding0.isEscalated());
+    assertFalse(outstanding2.isEscalated());
+
+    // two entries
+    assertEquals(2, escalations.size());
+    AbstractRMOperation e1 = escalations.get(0);
+    assertTrue(e1 instanceof CancelSingleRequest);
+    final CancelSingleRequest cancel = (CancelSingleRequest) e1;
+    assertEquals(initialRequest, cancel.getRequest());
+    AbstractRMOperation e2 = escalations.get(1);
+    assertTrue(e2 instanceof ContainerRequestOperation);
+    ContainerRequestOperation escRequest = (ContainerRequestOperation) e2;
+    assertTrue(escRequest.getRequest().getRelaxLocality());
+
+    // build that second request from an anti-affine entry
+    // these get placed as well
+    now += interval;
+    ContainerRequest containerReq2 =
+        outstanding2.buildContainerRequest(res2, getRole2Status(), now);
+    // escalate a little bit more
+    final List<AbstractRMOperation> escalations2 = tracker
+        .escalateOutstandingRequests(now);
+    // and expect no new entries
+    assertEquals(0, escalations2.size());
+
+    // go past the role2 timeout
+    now += getRole2Status().getPlacementTimeoutSeconds() * 1000 + 500;
+    // escalate a little bit more
+    final List<AbstractRMOperation> escalations3 = tracker
+        .escalateOutstandingRequests(now);
+    // and expect another escalation
+    assertEquals(2, escalations3.size());
+    assertTrue(outstanding2.isEscalated());
+
+    // finally add a strict entry to the mix
+    Resource res3 = newResource(getRole1Status());
+    OutstandingRequest outstanding3 = tracker.newRequest(host1,
+        getRole1Status().getKey());
+
+    final ProviderRole providerRole1 = getRole1Status().getProviderRole();
+    assertEquals(providerRole1.placementPolicy, PlacementPolicy.STRICT);
+    now += interval;
+    assertFalse(outstanding3.mayEscalate());
+    final List<AbstractRMOperation> escalations4 = tracker
+        .escalateOutstandingRequests(now);
+    assertTrue(escalations4.isEmpty());
+
+  }
+
+  /**
+   * If the placement does include a label, the initial request must
+   * <i>not</i> include it.
+   * The escalation request will contain the label, while
+   * leaving out the node list.
+   * retains the node list, but sets relaxLocality==true
+   * @throws Throwable
+   */
+  @Test
+  public void testRequestLabelledPlacement() throws Throwable {
+    NodeInstance ni = new NodeInstance("host1", 0);
+    OutstandingRequest req1 = tracker.newRequest(ni, 0);
+    Resource res0 = factory.newResource(48, 1);
+
+    RoleStatus workerRole = lookupRole(WORKER);
+    // initial request
+    ContainerRequest yarnRequest =
+        req1.buildContainerRequest(res0, workerRole, 0);
+    assertEquals(req1.label, WORKERS_LABEL);
+
+    assertNull(yarnRequest.getNodeLabelExpression());
+    assertFalse(yarnRequest.getRelaxLocality());
+    // escalation
+    ContainerRequest yarnRequest2 = req1.escalate();
+    assertNull(yarnRequest2.getNodes());
+    assertTrue(yarnRequest2.getRelaxLocality());
+    assertEquals(yarnRequest2.getNodeLabelExpression(), WORKERS_LABEL);
+  }
+
+  /**
+   * If the placement doesnt include a label, then the escalation request
+   * retains the node list, but sets relaxLocality==true.
+   * @throws Throwable
+   */
+  @Test
+  public void testRequestUnlabelledPlacement() throws Throwable {
+    NodeInstance ni = new NodeInstance("host1", 0);
+    OutstandingRequest req1 = tracker.newRequest(ni, 0);
+    Resource res0 = factory.newResource(48, 1);
+
+    // initial request
+    ContainerRequest yarnRequest = req1.buildContainerRequest(res0,
+        getRole0Status(), 0);
+    assertNotNull(yarnRequest.getNodes());
+    assertTrue(SliderUtils.isUnset(yarnRequest.getNodeLabelExpression()));
+    assertFalse(yarnRequest.getRelaxLocality());
+    ContainerRequest yarnRequest2 = req1.escalate();
+    assertNotNull(yarnRequest2.getNodes());
+    assertTrue(yarnRequest2.getRelaxLocality());
+  }
+
+  @Test(expected = IllegalArgumentException.class)
+  public void testAARequestNoNodes() throws Throwable {
+    tracker.newAARequest(getRole0Status().getKey(), new ArrayList<>(), "");
+  }
+
+  @Test
+  public void testAARequest() throws Throwable {
+    int role0 = getRole0Status().getKey();
+    OutstandingRequest request = tracker.newAARequest(role0, Arrays
+        .asList(host1), "");
+    assertEquals(host1.hostname, request.hostname);
+    assertFalse(request.isLocated());
+  }
+
+  @Test
+  public void testAARequestPair() throws Throwable {
+    int role0 = getRole0Status().getKey();
+    OutstandingRequest request = tracker.newAARequest(role0, Arrays.asList(
+        host1, host2), "");
+    assertEquals(host1.hostname, request.hostname);
+    assertFalse(request.isLocated());
+    ContainerRequest yarnRequest = request.buildContainerRequest(
+        getRole0Status().copyResourceRequirements(new MockResource(0, 0)),
+        getRole0Status(),
+        0);
+    assertFalse(yarnRequest.getRelaxLocality());
+    assertFalse(request.mayEscalate());
+
+    assertEquals(2, yarnRequest.getNodes().size());
+  }
+
+  @Test
+  public void testBuildResourceRequirements() throws Throwable {
+    // Store original values
+    Application application = appState.getClusterStatus();
+    Component role0 = application.getComponent(getRole0Status().getGroup());
+    String origMem = role0.getResource().getMemory();
+    Integer origVcores = role0.getResource().getCpus();
+
+    // Resource values to be used for this test
+    int testMem = 32768;
+    int testVcores = 2;
+    role0.resource(new org.apache.slider.api.resource.Resource().memory(Integer
+        .toString(testMem)).cpus(testVcores));
+
+    // Test normalization disabled
+    LOG.info("Test normalization: disabled");
+    role0.getConfiguration().setProperty(
+        ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "false");
+    MockResource requestedRes = new MockResource(testMem, testVcores);
+    MockResource expectedRes = new MockResource(testMem, testVcores);
+    LOG.info("Resource requested: {}", requestedRes);
+    Resource resFinal = appState.buildResourceRequirements(getRole0Status());
+    LOG.info("Resource actual: {}", resFinal);
+    assertTrue(Resources.equals(expectedRes, resFinal));
+
+    // Test normalization enabled
+    LOG.info("Test normalization: enabled");
+    role0.getConfiguration().setProperty(
+        ResourceKeys.YARN_RESOURCE_NORMALIZATION_ENABLED, "true");
+    expectedRes = new MockResource(MockAppState.RM_MAX_RAM, testVcores);
+    LOG.info("Resource requested: {}", requestedRes);
+    resFinal = appState.buildResourceRequirements(getRole0Status());
+    LOG.info("Resource actual: {}", resFinal);
+    assertTrue(Resources.equals(expectedRes, resFinal));
+
+    // revert resource configuration to original value
+    role0.resource(new org.apache.slider.api.resource.Resource().memory(origMem)
+        .cpus(origVcores));
+  }
+
+  public Resource newResource(RoleStatus r) {
+    return appState.buildResourceRequirements(r);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
new file mode 100644
index 0000000..a936df5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.java
@@ -0,0 +1,371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.avro.LoadedRoleHistory;
+import org.apache.slider.server.avro.RoleHistoryWriter;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test fole history reading and writing.
+ */
+public class TestRoleHistoryRW extends BaseMockAppStateTest {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestRoleHistoryRW.class);
+
+  private static long time = System.currentTimeMillis();
+  public static final String HISTORY_V1_6_ROLE =
+      "org/apache/slider/server/avro/history-v01-6-role.json";
+  public static final String HISTORY_V1_3_ROLE =
+      "org/apache/slider/server/avro/history-v01-3-role.json";
+  public static final String HISTORY_V1B_1_ROLE =
+      "org/apache/slider/server/avro/history_v01b_1_role.json";
+
+  private RoleStatus role0Status;
+  private RoleStatus role1Status;
+
+  static final ProviderRole PROVIDER_ROLE3 = new ProviderRole(
+      "role3",
+      3,
+      PlacementPolicy.STRICT,
+      3,
+      3,
+      ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
+
+  @Override
+  public String getTestName() {
+    return "TestHistoryRW";
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    role0Status = getRole0Status();
+    role1Status = getRole1Status();
+  }
+
+  @Test
+  public void testWriteReadEmpty() throws Throwable {
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    roleHistory.onStart(fs, historyPath);
+    Path history = roleHistory.saveHistory(time++);
+    assertTrue(fs.getFileStatus(history).isFile());
+    RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+    historyWriter.read(fs, history);
+  }
+
+  @Test
+  public void testWriteReadData() throws Throwable {
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    assertFalse(roleHistory.onStart(fs, historyPath));
+    String addr = "localhost";
+    NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+    NodeEntry ne1 = instance.getOrCreate(0);
+    ne1.setLastUsed(0xf00d);
+
+    Path history = roleHistory.saveHistory(time++);
+    assertTrue(fs.getFileStatus(history).isFile());
+    RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+    RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+
+
+    LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history);
+    assertTrue(0 < loadedRoleHistory.size());
+    rh2.rebuild(loadedRoleHistory);
+    NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+    assertNotNull(ni2);
+    NodeEntry ne2 = ni2.get(0);
+    assertNotNull(ne2);
+    assertEquals(ne2.getLastUsed(), ne1.getLastUsed());
+  }
+
+  @Test
+  public void testWriteReadActiveData() throws Throwable {
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    roleHistory.onStart(fs, historyPath);
+    String addr = "localhost";
+    String addr2 = "rack1server5";
+    NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr);
+    NodeEntry orig1 = localhost.getOrCreate(role0Status.getKey());
+    orig1.setLastUsed(0x10);
+    NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2);
+    NodeEntry orig2 = rack1server5.getOrCreate(role1Status.getKey());
+    orig2.setLive(3);
+    assertFalse(orig2.isAvailable());
+    NodeEntry orig3 = localhost.getOrCreate(role1Status.getKey());
+    orig3.setLastUsed(0x20);
+    orig3.setLive(1);
+    assertFalse(orig3.isAvailable());
+    orig3.release();
+    assertTrue(orig3.isAvailable());
+    roleHistory.dump();
+
+    long savetime = 0x0001000;
+    Path history = roleHistory.saveHistory(savetime);
+    assertTrue(fs.getFileStatus(history).isFile());
+    describe("Loaded");
+    LOG.info("testWriteReadActiveData in {}", history);
+    RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+    RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+    LoadedRoleHistory loadedRoleHistory = historyWriter.read(fs, history);
+    assertEquals(3, loadedRoleHistory.size());
+    rh2.rebuild(loadedRoleHistory);
+    rh2.dump();
+
+    assertEquals(2, rh2.getClusterSize());
+    NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+    assertNotNull(ni2);
+    NodeEntry loadedNE = ni2.get(role0Status.getKey());
+    assertEquals(loadedNE.getLastUsed(), orig1.getLastUsed());
+    NodeInstance ni2b = rh2.getExistingNodeInstance(addr2);
+    assertNotNull(ni2b);
+    NodeEntry loadedNE2 = ni2b.get(role1Status.getKey());
+    assertNotNull(loadedNE2);
+    assertEquals(loadedNE2.getLastUsed(), savetime);
+    assertEquals(rh2.getThawedDataTime(), savetime);
+
+    // now start it
+    rh2.buildRecentNodeLists();
+    describe("starting");
+    rh2.dump();
+    List<NodeInstance> available0 = rh2.cloneRecentNodeList(role0Status
+        .getKey());
+    assertEquals(1, available0.size());
+
+    NodeInstance entry = available0.get(0);
+    assertEquals(entry.hostname, "localhost");
+    assertEquals(entry, localhost);
+    List<NodeInstance> available1 = rh2.cloneRecentNodeList(role1Status
+        .getKey());
+    assertEquals(2, available1.size());
+    //and verify that even if last used was set, the save time is picked up
+    assertEquals(entry.get(role1Status.getKey()).getLastUsed(), roleHistory
+        .getSaveTime());
+
+  }
+
+  @Test
+  public void testWriteThaw() throws Throwable {
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    assertFalse(roleHistory.onStart(fs, historyPath));
+    String addr = "localhost";
+    NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+    NodeEntry ne1 = instance.getOrCreate(0);
+    ne1.setLastUsed(0xf00d);
+
+    Path history = roleHistory.saveHistory(time++);
+    long savetime =roleHistory.getSaveTime();
+    assertTrue(fs.getFileStatus(history).isFile());
+    RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+    assertTrue(rh2.onStart(fs, historyPath));
+    NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+    assertNotNull(ni2);
+    NodeEntry ne2 = ni2.get(0);
+    assertNotNull(ne2);
+    assertEquals(ne2.getLastUsed(), ne1.getLastUsed());
+    assertEquals(rh2.getThawedDataTime(), savetime);
+  }
+
+
+  @Test
+  public void testPurgeOlderEntries() throws Throwable {
+    RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+    time = 1;
+    Path file1 = touch(historyWriter, time++);
+    Path file2 = touch(historyWriter, time++);
+    Path file3 = touch(historyWriter, time++);
+    Path file4 = touch(historyWriter, time++);
+    Path file5 = touch(historyWriter, time++);
+    Path file6 = touch(historyWriter, time++);
+
+    assertEquals(0, historyWriter.purgeOlderHistoryEntries(fs, file1));
+    assertEquals(1, historyWriter.purgeOlderHistoryEntries(fs, file2));
+    assertEquals(0, historyWriter.purgeOlderHistoryEntries(fs, file2));
+    assertEquals(3, historyWriter.purgeOlderHistoryEntries(fs, file5));
+    assertEquals(1, historyWriter.purgeOlderHistoryEntries(fs, file6));
+    try {
+      // make an impossible assertion that will fail if the method
+      // actually completes
+      assertEquals(-1, historyWriter.purgeOlderHistoryEntries(fs, file1));
+    } catch (FileNotFoundException ignored) {
+      //  expected
+    }
+
+  }
+
+  public Path touch(RoleHistoryWriter historyWriter, long timeMs)
+      throws IOException {
+    Path path = historyWriter.createHistoryFilename(historyPath, timeMs);
+    FSDataOutputStream out = fs.create(path);
+    out.close();
+    return path;
+  }
+
+  @Test
+  public void testSkipEmptyFileOnRead() throws Throwable {
+    describe("verify that empty histories are skipped on read; old histories " +
+            "purged");
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    roleHistory.onStart(fs, historyPath);
+    time = 0;
+    Path oldhistory = roleHistory.saveHistory(time++);
+
+    String addr = "localhost";
+    NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+    NodeEntry ne1 = instance.getOrCreate(0);
+    ne1.setLastUsed(0xf00d);
+
+    Path goodhistory = roleHistory.saveHistory(time++);
+
+    RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+    Path touched = touch(historyWriter, time++);
+
+    RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+    assertTrue(rh2.onStart(fs, historyPath));
+    NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+    assertNotNull(ni2);
+
+    //and assert the older file got purged
+    assertFalse(fs.exists(oldhistory));
+    assertTrue(fs.exists(goodhistory));
+    assertTrue(fs.exists(touched));
+  }
+
+  @Test
+  public void testSkipBrokenFileOnRead() throws Throwable {
+    describe("verify that empty histories are skipped on read; old histories " +
+            "purged");
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    roleHistory.onStart(fs, historyPath);
+    time = 0;
+    Path oldhistory = roleHistory.saveHistory(time++);
+
+    String addr = "localhost";
+    NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+    NodeEntry ne1 = instance.getOrCreate(0);
+    ne1.setLastUsed(0xf00d);
+
+    Path goodhistory = roleHistory.saveHistory(time++);
+
+    RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+    Path badfile = historyWriter.createHistoryFilename(historyPath, time++);
+    FSDataOutputStream out = fs.create(badfile);
+    out.writeBytes("{broken:true}");
+    out.close();
+
+    RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES);
+    describe("IGNORE STACK TRACE BELOW");
+
+    assertTrue(rh2.onStart(fs, historyPath));
+
+    describe("IGNORE STACK TRACE ABOVE");
+    NodeInstance ni2 = rh2.getExistingNodeInstance(addr);
+    assertNotNull(ni2);
+
+    //and assert the older file got purged
+    assertFalse(fs.exists(oldhistory));
+    assertTrue(fs.exists(goodhistory));
+    assertTrue(fs.exists(badfile));
+  }
+
+  /**
+   * Test that a v1 JSON file can be read. Here the number of roles
+   * matches the current state.
+   * @throws Throwable
+   */
+  @Test
+  public void testReloadDataV13Role() throws Throwable {
+    String source = HISTORY_V1_3_ROLE;
+    RoleHistoryWriter writer = new RoleHistoryWriter();
+
+    LoadedRoleHistory loadedRoleHistory = writer.read(source);
+    assertEquals(4, loadedRoleHistory.size());
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
+  }
+
+  /**
+   * Test that a v1 JSON file can be read. Here more roles than expected
+   * @throws Throwable
+   */
+  @Test
+  public void testReloadDataV16Role() throws Throwable {
+    String source = HISTORY_V1_6_ROLE;
+    RoleHistoryWriter writer = new RoleHistoryWriter();
+
+    LoadedRoleHistory loadedRoleHistory = writer.read(source);
+    assertEquals(6, loadedRoleHistory.size());
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    assertEquals(3, roleHistory.rebuild(loadedRoleHistory));
+  }
+
+  /**
+   * Test that a v1 JSON file can be read. Here the number of roles
+   * is less than the current state.
+   * @throws Throwable
+   */
+  @Test
+  public void testReloadLessRoles() throws Throwable {
+    String source = HISTORY_V1_3_ROLE;
+    RoleHistoryWriter writer = new RoleHistoryWriter();
+
+    LoadedRoleHistory loadedRoleHistory = writer.read(source);
+    assertEquals(4, loadedRoleHistory.size());
+    List<ProviderRole> expandedRoles = new ArrayList(MockFactory.ROLES);
+    expandedRoles.add(PROVIDER_ROLE3);
+    RoleHistory roleHistory = new MockRoleHistory(expandedRoles);
+    assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
+  }
+
+  /**
+   * Test that a v1b JSON file can be read. Here more roles than expected
+   * @throws Throwable
+   */
+  @Test
+  public void testReloadDataV1B1Role() throws Throwable {
+    String source = HISTORY_V1B_1_ROLE;
+    RoleHistoryWriter writer = new RoleHistoryWriter();
+
+    LoadedRoleHistory loadedRoleHistory = writer.read(source);
+    assertEquals(1, loadedRoleHistory.size());
+    assertEquals(2, loadedRoleHistory.roleMap.size());
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    assertEquals(0, roleHistory.rebuild(loadedRoleHistory));
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
new file mode 100644
index 0000000..0bc2282
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.avro.NewerFilesFirst;
+import org.apache.slider.server.avro.RoleHistoryWriter;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Test role history rw ordering.
+ */
+public class TestRoleHistoryRWOrdering extends BaseMockAppStateTest {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestRoleHistoryRWOrdering.class);
+
+  private List<Path> paths = pathlist(
+      Arrays.asList(
+        "hdfs://localhost/history-0406c.json",
+        "hdfs://localhost/history-5fffa.json",
+        "hdfs://localhost/history-0001a.json",
+        "hdfs://localhost/history-0001f.json"
+      )
+  );
+  private Path h0406c = paths.get(0);
+  private Path h5fffa = paths.get(1);
+  private Path h0001a = paths.get(3);
+
+  public TestRoleHistoryRWOrdering() throws URISyntaxException {
+  }
+
+  List<Path> pathlist(List<String> pathnames) throws URISyntaxException {
+    List<Path> pathList = new ArrayList<>();
+    for (String p : pathnames) {
+      pathList.add(new Path(new URI(p)));
+    }
+    return pathList;
+  }
+
+  @Override
+  public String getTestName() {
+    return "TestHistoryRWOrdering";
+  }
+
+  /**
+   * This tests regexp pattern matching. It uses the current time so isn't
+   * repeatable -but it does test a wider range of values in the process
+   * @throws Throwable
+   */
+  @Test
+  public void testPatternRoundTrip() throws Throwable {
+    describe("test pattern matching of names");
+    long value=System.currentTimeMillis();
+    String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,
+        value);
+    String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN;
+    Pattern pattern = Pattern.compile(matchpattern);
+    Matcher matcher = pattern.matcher(name);
+    if (!matcher.find()) {
+      throw new Exception("No match for pattern $matchpattern in $name");
+    }
+  }
+
+  @Test
+  public void testWriteSequenceReadData() throws Throwable {
+    describe("test that if multiple entries are written, the newest is picked" +
+        " up");
+    long time = System.currentTimeMillis();
+
+    RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+    assertFalse(roleHistory.onStart(fs, historyPath));
+    String addr = "localhost";
+    NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr);
+    NodeEntry ne1 = instance.getOrCreate(0);
+    ne1.setLastUsed(0xf00d);
+
+    Path history1 = roleHistory.saveHistory(time++);
+    Path history2 = roleHistory.saveHistory(time++);
+    Path history3 = roleHistory.saveHistory(time);
+
+    //inject a later file with a different name
+    sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello," +
+        " world");
+
+
+    RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+
+    List<Path> entries = historyWriter.findAllHistoryEntries(
+        fs,
+        historyPath,
+        false);
+    assertEquals(entries.size(), 3);
+    assertEquals(entries.get(0), history3);
+    assertEquals(entries.get(1), history2);
+    assertEquals(entries.get(2), history1);
+  }
+
+  @Test
+  public void testPathStructure() throws Throwable {
+    assertEquals(h5fffa.getName(), "history-5fffa.json");
+  }
+
+  @Test
+  public void testPathnameComparator() throws Throwable {
+
+    NewerFilesFirst newerName = new NewerFilesFirst();
+
+    LOG.info("{} name is {}", h5fffa, h5fffa.getName());
+    LOG.info("{} name is {}", h0406c, h0406c.getName());
+    assertEquals(newerName.compare(h5fffa, h5fffa), 0);
+    assertTrue(newerName.compare(h5fffa, h0406c) < 0);
+    assertTrue(newerName.compare(h5fffa, h0001a) < 0);
+    assertTrue(newerName.compare(h0001a, h5fffa) > 0);
+
+  }
+
+  @Test
+  public void testPathSort() throws Throwable {
+    List<Path> paths2 = new ArrayList<>(paths);
+    RoleHistoryWriter.sortHistoryPaths(paths2);
+    assertListEquals(paths2,
+                     Arrays.asList(
+                       paths.get(1),
+                       paths.get(0),
+                       paths.get(3),
+                       paths.get(2)
+                     ));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
new file mode 100644
index 0000000..7364201
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockContainer;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.OutstandingRequest;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test the RH availability list and request tracking: that hosts
+ * get removed and added.
+ */
+public class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestRoleHistoryRequestTracking.class);
+
+  private String roleName = "test";
+
+  private NodeInstance age1Active4;
+  private NodeInstance age2Active2;
+  private NodeInstance age2Active0;
+  private NodeInstance age3Active0;
+  private NodeInstance age4Active1;
+
+  private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+  // 1MB, 1 vcore
+  private Resource resource = Resource.newInstance(1, 1);
+
+  private RoleStatus roleStatus;
+
+  public TestRoleHistoryRequestTracking() throws BadConfigException {
+  }
+
+  AMRMClient.ContainerRequest requestContainer(RoleStatus rs) {
+    return roleHistory.requestContainerForRole(rs).getIssuedRequest();
+  }
+
+  @Override
+  public String getTestName() {
+    return "TestRoleHistoryAvailableList";
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+
+    age1Active4 = nodeInstance(1, 4, 0, 0);
+    age2Active2 = nodeInstance(2, 2, 0, 1);
+    age2Active0 = nodeInstance(2, 0, 0, 0);
+    age3Active0 = nodeInstance(3, 0, 0, 0);
+    age4Active1 = nodeInstance(4, 1, 0, 0);
+
+    roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1,
+        age1Active4, age3Active0));
+    roleHistory.buildRecentNodeLists();
+    roleStatus = getRole0Status();
+    roleStatus.setResourceRequirements(Resource.newInstance(1, 1));
+  }
+
+  @Test
+  public void testAvailableListBuiltForRoles() throws Throwable {
+    List<NodeInstance> available0 = roleHistory.cloneRecentNodeList(
+        roleStatus.getKey());
+    assertListEquals(Arrays.asList(age3Active0, age2Active0), available0);
+  }
+
+  @Test
+  public void testRequestedNodeOffList() throws Throwable {
+    NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
+    assertEquals(age3Active0, ni);
+    assertListEquals(Arrays.asList(age2Active0),
+        roleHistory.cloneRecentNodeList(roleStatus.getKey()));
+    roleHistory.requestInstanceOnNode(ni,
+        roleStatus,
+        resource
+    );
+  }
+
+  @Test
+  public void testRequestedNodeOffListWithFailures() throws Throwable {
+    assertFalse(roleHistory.cloneRecentNodeList(roleStatus.getKey()).isEmpty());
+
+    NodeEntry age3role0 = recordAsFailed(age3Active0, roleStatus.getKey(), 4);
+    assertTrue(age3Active0.isConsideredUnreliable(roleStatus.getKey(),
+        roleStatus.getNodeFailureThreshold()));
+    recordAsFailed(age2Active0, roleStatus.getKey(), 4);
+    assertTrue(age2Active0.isConsideredUnreliable(roleStatus.getKey(),
+        roleStatus.getNodeFailureThreshold()));
+    // expect to get a null node back
+    NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
+    assertNull(ni);
+
+    // which is translated to a no-location request
+    AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
+        roleStatus,
+        resource).getIssuedRequest();
+
+    assertNull(req.getNodes());
+
+    LOG.info("resetting failure count");
+    age3role0.resetFailedRecently();
+    roleHistory.dump();
+    assertEquals(0, age3role0.getFailedRecently());
+    assertFalse(age3Active0.isConsideredUnreliable(roleStatus.getKey(),
+        roleStatus.getNodeFailureThreshold()));
+    assertFalse(roleHistory.cloneRecentNodeList(roleStatus.getKey()).isEmpty());
+    // looking for a node should now find one
+    ni = roleHistory.findRecentNodeForNewInstance(roleStatus);
+    assertEquals(ni, age3Active0);
+    req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource)
+        .getIssuedRequest();
+    assertEquals(1, req.getNodes().size());
+  }
+
+  /**
+   * Verify that strict placement policies generate requests for nodes
+   * irrespective of their failed status.
+   * @throws Throwable
+   */
+  @Test
+  public void testStrictPlacementIgnoresFailures() throws Throwable {
+
+    RoleStatus targetRole = getRole1Status();
+    final ProviderRole providerRole1 = targetRole.getProviderRole();
+    assertEquals(providerRole1.placementPolicy, PlacementPolicy.STRICT);
+    int key1 = targetRole.getKey();
+    int key0 = getRole0Status().getKey();
+
+    List<NodeInstance> nodes0 = Arrays.asList(age1Active4, age2Active0,
+        age2Active2, age3Active0, age4Active1);
+    recordAllFailed(key0, 4, nodes0);
+    recordAllFailed(key1, 4, nodes0);
+
+    // trigger a list rebuild
+    roleHistory.buildRecentNodeLists();
+    List<NodeInstance> recentRole0 = roleHistory.cloneRecentNodeList(key0);
+    assertTrue(recentRole0.indexOf(age3Active0) < recentRole0
+        .indexOf(age2Active0));
+
+    // the non-strict role has no suitable nodes
+    assertNull(roleHistory.findRecentNodeForNewInstance(getRole0Status()));
+
+
+    NodeInstance ni = roleHistory.findRecentNodeForNewInstance(targetRole);
+    assertNotNull(ni);
+
+    NodeInstance ni2 = roleHistory.findRecentNodeForNewInstance(targetRole);
+    assertNotNull(ni2);
+    assertNotEquals(ni, ni2);
+  }
+
+  @Test
+  public void testFindAndRequestNode() throws Throwable {
+    AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+
+    assertEquals(age3Active0.hostname, req.getNodes().get(0));
+    List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(roleStatus
+        .getKey());
+    assertListEquals(Arrays.asList(age2Active0), a2);
+  }
+
+  @Test
+  public void testRequestedNodeIntoReqList() throws Throwable {
+    requestContainer(roleStatus);
+    List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
+    assertEquals(1, requests.size());
+    assertEquals(age3Active0.hostname, requests.get(0).hostname);
+  }
+
+  @Test
+  public void testCompletedRequestDropsNode() throws Throwable {
+    AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+    List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
+    assertEquals(1, requests.size());
+    String hostname = requests.get(0).hostname;
+    assertEquals(age3Active0.hostname, hostname);
+    assertEquals(hostname, req.getNodes().get(0));
+    MockContainer container = factory.newContainer(req, hostname);
+    assertOnContainerAllocated(container, 2, 1);
+    assertNoOutstandingPlacedRequests();
+  }
+
+  public void assertOnContainerAllocated(Container c1, int p1, int p2) {
+    assertNotEquals(ContainerAllocationOutcome.Open, roleHistory
+        .onContainerAllocated(c1, p1, p2).outcome);
+  }
+
+  public void assertOnContainerAllocationOpen(Container c1, int p1, int p2) {
+    assertEquals(ContainerAllocationOutcome.Open, roleHistory
+        .onContainerAllocated(c1, p1, p2).outcome);
+  }
+
+  void assertNoOutstandingPlacedRequests() {
+    assertTrue(roleHistory.listPlacedRequests().isEmpty());
+  }
+
+  public void assertOutstandingPlacedRequests(int i) {
+    assertEquals(i, roleHistory.listPlacedRequests().size());
+  }
+
+  @Test
+  public void testTwoRequests() throws Throwable {
+    AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+    AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
+    List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
+    assertEquals(2, requests.size());
+    MockContainer container = factory.newContainer(req, req.getNodes().get(0));
+    assertOnContainerAllocated(container, 2, 1);
+    assertOutstandingPlacedRequests(1);
+    container = factory.newContainer(req2, req2.getNodes().get(0));
+    assertOnContainerAllocated(container, 2, 2);
+    assertNoOutstandingPlacedRequests();
+  }
+
+  @Test
+  public void testThreeRequestsOneUnsatisified() throws Throwable {
+    AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+    AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
+    AMRMClient.ContainerRequest req3 = requestContainer(roleStatus);
+    List<OutstandingRequest> requests = roleHistory.listPlacedRequests();
+    assertEquals(2, requests.size());
+    MockContainer container = factory.newContainer(req, req.getNodes().get(0));
+    assertOnContainerAllocated(container, 2, 1);
+    assertOutstandingPlacedRequests(1);
+
+    container = factory.newContainer(req3, "three");
+    assertOnContainerAllocationOpen(container, 3, 2);
+    assertOutstandingPlacedRequests(1);
+
+    // the final allocation will trigger a cleanup
+    container = factory.newContainer(req2, "four");
+    // no node dropped
+    assertEquals(ContainerAllocationOutcome.Unallocated,
+           roleHistory.onContainerAllocated(container, 3, 3).outcome);
+    // yet the list is now empty
+    assertNoOutstandingPlacedRequests();
+    roleHistory.listOpenRequests().isEmpty();
+
+    // and the remainder goes onto the available list
+    List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(roleStatus
+        .getKey());
+    assertListEquals(Arrays.asList(age2Active0), a2);
+  }
+
+  @Test
+  public void testThreeRequests() throws Throwable {
+    AMRMClient.ContainerRequest req = requestContainer(roleStatus);
+    AMRMClient.ContainerRequest req2 = requestContainer(roleStatus);
+    AMRMClient.ContainerRequest req3 = requestContainer(roleStatus);
+    assertOutstandingPlacedRequests(2);
+    assertNull(req3.getNodes());
+    MockContainer container = factory.newContainer(req, req.getNodes().get(0));
+    assertOnContainerAllocated(container, 3, 1);
+    assertOutstandingPlacedRequests(1);
+    container = factory.newContainer(req2, req2.getNodes().get(0));
+    assertOnContainerAllocated(container, 3, 2);
+    assertNoOutstandingPlacedRequests();
+    container = factory.newContainer(req3, "three");
+    assertOnContainerAllocationOpen(container, 3, 3);
+    assertNoOutstandingPlacedRequests();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
new file mode 100644
index 0000000..33e7930
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryUpdateBlacklist.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history;
+
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
+import org.apache.slider.server.appmaster.model.mock.MockAM;
+import org.apache.slider.server.appmaster.model.mock.MockFactory;
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler;
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.RoleHistory;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Test updating blacklist.
+ */
+public class TestRoleHistoryUpdateBlacklist extends BaseMockAppStateTest {
+  private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
+  private Collection<RoleStatus> roleStatuses;
+  private RoleStatus roleStatus;
+  private NodeInstance ni;
+
+  public TestRoleHistoryUpdateBlacklist() throws BadConfigException {
+  }
+
+  @Override
+  public String getTestName() {
+    return "TestUpdateBlacklist";
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    ni = nodeInstance(1, 0, 0, 0);
+    roleHistory.insert(Arrays.asList(ni));
+    roleHistory.buildRecentNodeLists();
+    appState.setRoleHistory(roleHistory);
+    roleStatus = getRole0Status();
+    roleStatuses = Arrays.asList(roleStatus);
+  }
+
+  @Test
+  public void testUpdateBlacklist() {
+    assertFalse(ni.isBlacklisted());
+
+    // at threshold, blacklist is unmodified
+    recordAsFailed(ni, roleStatus.getKey(), MockFactory.NODE_FAILURE_THRESHOLD);
+    UpdateBlacklistOperation op = roleHistory.updateBlacklist(roleStatuses);
+    assertNull(op);
+    assertFalse(ni.isBlacklisted());
+
+    // threshold is reached, node goes on blacklist
+    recordAsFailed(ni, roleStatus.getKey(), 1);
+    op = roleHistory.updateBlacklist(roleStatuses);
+    assertNotNull(op);
+    assertTrue(ni.isBlacklisted());
+
+    // blacklist remains unmodified
+    op = roleHistory.updateBlacklist(roleStatuses);
+    assertNull(op);
+    assertTrue(ni.isBlacklisted());
+
+    // failure threshold reset, node goes off blacklist
+    ni.resetFailedRecently();
+    op = roleHistory.updateBlacklist(roleStatuses);
+    assertNotNull(op);
+    assertFalse(ni.isBlacklisted());
+  }
+
+  @Test
+  public void testBlacklistOperations()
+      throws Exception {
+    recordAsFailed(ni, roleStatus.getKey(), MockFactory
+        .NODE_FAILURE_THRESHOLD + 1);
+
+    List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+    assertListLength(ops, 1);
+    AbstractRMOperation op = ops.get(0);
+    assertTrue(op instanceof UpdateBlacklistOperation);
+    assertTrue(ni.isBlacklisted());
+
+    MockRMOperationHandler handler = new MockRMOperationHandler();
+    assertEquals(0, handler.getBlacklisted());
+    handler.execute(ops);
+    assertEquals(1, handler.getBlacklisted());
+
+    ResetFailureWindow resetter = new ResetFailureWindow(handler);
+    resetter.execute(new MockAM(), null, appState);
+    assertEquals(0, handler.getBlacklisted());
+    assertFalse(ni.isBlacklisted());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
new file mode 100644
index 0000000..419f2fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/Allocator.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.common.tools.SliderUtils;
+
+/**
+ * Provides allocation services to a cluster -both random and placed.
+ *
+ * Important: container allocations need an app attempt ID put into the
+ * container ID
+ */
+public class Allocator {
+
+  private final MockYarnCluster cluster;
+  /**
+   * Rolling index into the cluster used for the next "random" assignment.
+   */
+  private int rollingIndex = 0;
+
+  Allocator(MockYarnCluster cluster) {
+    this.cluster = cluster;
+  }
+
+  /**
+   * Allocate a node using the list of nodes in the container as the
+   * hints.
+   * @param request request
+   * @return the allocated container -or null for none
+   */
+  MockContainer allocate(AMRMClient.ContainerRequest request) {
+    MockYarnCluster.MockYarnClusterNode node = null;
+    MockYarnCluster.MockYarnClusterContainer allocated = null;
+    if (SliderUtils.isNotEmpty(request.getNodes())) {
+      for (String host : request.getNodes()) {
+        node = cluster.lookup(host);
+        allocated = node.allocate();
+        if (allocated != null) {
+          break;
+        }
+      }
+    }
+
+    if (allocated != null) {
+      return createContainerRecord(request, allocated, node);
+    } else {
+      if (request.getRelaxLocality() || request.getNodes().isEmpty()) {
+        // fallback to anywhere
+        return allocateRandom(request);
+      } else {
+        //no match and locality can't be requested
+        return null;
+      }
+    }
+  }
+
+  /**
+   * Allocate a node without any positioning -use whatever policy this allocator
+   * chooses.
+   * @param request request
+   * @return the allocated container -or null for none
+   */
+  MockContainer allocateRandom(AMRMClient.ContainerRequest request) {
+    int start = rollingIndex;
+    MockYarnCluster.MockYarnClusterNode node = cluster.nodeAt(rollingIndex);
+    MockYarnCluster.MockYarnClusterContainer allocated = node.allocate();
+    // if there is no space, try again -but stop when all the nodes
+    // have failed
+    while (allocated == null && start != nextIndex()) {
+      node = cluster.nodeAt(rollingIndex);
+      allocated = node.allocate();
+    }
+
+    //here the allocation is set, so create the response
+    return createContainerRecord(request, allocated, node);
+  }
+
+  /**
+   * Create a container record -if one was allocated.
+   * @param allocated allocation -may be null
+   * @param node node with the container
+   * @return a container record, or null if there was no allocation
+   */
+  public MockContainer createContainerRecord(
+      AMRMClient.ContainerRequest request,
+      MockYarnCluster.MockYarnClusterContainer allocated,
+      MockYarnCluster.MockYarnClusterNode node) {
+    if (allocated == null) {
+      // no space
+      return null;
+    }
+    MockContainer container = new MockContainer();
+    container.setId(new MockContainerId(allocated.getCid()));
+    container.setNodeId(node.getNodeId());
+    container.setNodeHttpAddress(node.httpAddress());
+    container.setPriority(request.getPriority());
+    container.setResource(request.getCapability());
+    return container;
+  }
+
+  public int nextIndex() {
+    rollingIndex = (rollingIndex + 1) % cluster.getClusterSize();
+    return rollingIndex;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
new file mode 100644
index 0000000..eca8401
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
@@ -0,0 +1,524 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
+import org.apache.slider.server.appmaster.state.ContainerAssignment;
+import org.apache.slider.server.appmaster.state.ContainerOutcome;
+import org.apache.slider.server.appmaster.state.NodeEntry;
+import org.apache.slider.server.appmaster.state.NodeInstance;
+import org.apache.slider.server.appmaster.state.NodeMap;
+import org.apache.slider.server.appmaster.state.ProviderAppState;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.utils.SliderTestBase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+
+/**
+ * Base for app state tests.
+ */
+public abstract class BaseMockAppStateTest extends SliderTestBase implements
+    MockRoles {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(BaseMockAppStateTest.class);
+  protected static final List<ContainerId> EMPTY_ID_LIST = Collections
+      .emptyList();
+
+  protected final MockFactory factory = MockFactory.INSTANCE;
+  protected MockAppState appState;
+  protected MockYarnEngine engine;
+  protected FileSystem fs;
+  protected SliderFileSystem sliderFileSystem;
+  protected File historyWorkDir;
+  protected Path historyPath;
+  protected MockApplicationId applicationId;
+  protected MockApplicationAttemptId applicationAttemptId;
+  protected StateAccessForProviders stateAccess;
+
+  /**
+   * Override point: called in setup() to create the YARN engine; can
+   * be changed for different sizes and options.
+   * @return
+   */
+  public MockYarnEngine createYarnEngine() {
+    return new MockYarnEngine(8, 8);
+  }
+
+  @Override
+  public void setup() throws Exception {
+    super.setup();
+    YarnConfiguration conf = SliderUtils.createConfiguration();
+    fs = FileSystem.get(new URI("file:///"), conf);
+    sliderFileSystem = new SliderFileSystem(fs, conf);
+    engine = createYarnEngine();
+    initApp();
+  }
+
+  /**
+   * Initialize the application.
+   * This uses the binding information supplied by {@link #buildBindingInfo()}.
+   */
+  protected void initApp()
+      throws IOException, BadConfigException, BadClusterStateException {
+    String historyDirName = getTestName();
+    applicationId = new MockApplicationId(1, 0);
+    applicationAttemptId = new MockApplicationAttemptId(applicationId, 1);
+
+    historyWorkDir = new File("target/history", historyDirName);
+    historyPath = new Path(historyWorkDir.toURI());
+    fs.delete(historyPath, true);
+    appState = new MockAppState(buildBindingInfo());
+    stateAccess = new ProviderAppState(getTestName(), appState);
+  }
+
+  /**
+   * Build the binding info from the default constructor values,
+   * the roles from {@link #factory}, and an instance definition.
+   * from {@link #buildApplication()} ()}
+   * @return
+   */
+  protected AppStateBindingInfo buildBindingInfo() {
+    AppStateBindingInfo binding = new AppStateBindingInfo();
+    binding.application = buildApplication();
+    //binding.roles = new ArrayList<>(factory.ROLES);
+    binding.fs = fs;
+    binding.historyPath = historyPath;
+    binding.nodeReports = engine.getNodeReports();
+    return binding;
+  }
+
+  /**
+   * Override point, define the instance definition.
+   * @return the instance definition
+   */
+  public Application buildApplication() {
+    return factory.newApplication(0, 0, 0).name(getTestName());
+  }
+
+  /**
+   * Get the test name ... defaults to method name
+   * @return the method name
+   */
+  public String getTestName() {
+    return methodName.getMethodName();
+  }
+
+  public RoleStatus getRole0Status() {
+    return lookupRole(ROLE0);
+  }
+
+  public RoleStatus lookupRole(String role) {
+    return appState.lookupRoleStatus(role);
+  }
+
+  public RoleStatus getRole1Status() {
+    return lookupRole(ROLE1);
+  }
+
+  public RoleStatus getRole2Status() {
+    return lookupRole(ROLE2);
+  }
+
+  /**
+   * Build a role instance from a container assignment.
+   * @param assigned
+   * @return the instance
+   */
+  public RoleInstance roleInstance(ContainerAssignment assigned) {
+    Container target = assigned.container;
+    RoleInstance ri = new RoleInstance(target);
+    ri.roleId = assigned.role.getPriority();
+    ri.role = assigned.role.getName();
+    return ri;
+  }
+
+  public NodeInstance nodeInstance(long age, int live0, int live1, int live2) {
+    NodeInstance ni = new NodeInstance(String.format("age%d-[%d,%d,%d]", age,
+        live0, live1, live2), MockFactory.ROLE_COUNT);
+    ni.getOrCreate(getRole0Status().getKey()).setLastUsed(age);
+    ni.getOrCreate(getRole0Status().getKey()).setLive(live0);
+    if (live1 > 0) {
+      ni.getOrCreate(getRole1Status().getKey()).setLive(live1);
+    }
+    if (live2 > 0) {
+      ni.getOrCreate(getRole2Status().getKey()).setLive(live2);
+    }
+    return ni;
+  }
+
+  /**
+   * Create a container status event.
+   * @param c container
+   * @return a status
+   */
+  ContainerStatus containerStatus(Container c) {
+    return containerStatus(c.getId());
+  }
+
+  /**
+   * Create a container status instance for the given ID, declaring
+   * that it was shut down by the application itself.
+   * @param cid container Id
+   * @return the instance
+   */
+  public ContainerStatus containerStatus(ContainerId cid) {
+    ContainerStatus status = containerStatus(cid,
+        LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN);
+    return status;
+  }
+
+  public ContainerStatus containerStatus(ContainerId cid, int exitCode) {
+    ContainerStatus status = ContainerStatus.newInstance(
+        cid,
+        ContainerState.COMPLETE,
+        "",
+        exitCode);
+    return status;
+  }
+
+  /**
+   * Create nodes and bring them to the started state.
+   * @return a list of roles
+   */
+  protected List<RoleInstance> createAndStartNodes()
+      throws TriggerClusterTeardownException, SliderInternalStateException {
+    return createStartAndStopNodes(new ArrayList<>());
+  }
+
+  /**
+   * Create, Start and stop nodes.
+   * @param completionResults List filled in with the status on all completed
+   *                          nodes
+   * @return the nodes
+   */
+  public List<RoleInstance> createStartAndStopNodes(
+      List<AppState.NodeCompletionResult> completionResults)
+      throws TriggerClusterTeardownException, SliderInternalStateException {
+    List<ContainerId> released = new ArrayList<>();
+    List<RoleInstance> instances = createAndSubmitNodes(released);
+    processSubmissionOperations(instances, completionResults, released);
+    return instances;
+  }
+
+  /**
+   * Process the start/stop operations.
+   * @param instances
+   * @param completionResults
+   * @param released
+   */
+  public void processSubmissionOperations(
+      List<RoleInstance> instances,
+      List<AppState.NodeCompletionResult> completionResults,
+      List<ContainerId> released) {
+    for (RoleInstance instance : instances) {
+      LOG.debug("Started {} on {}", instance.role, instance.id);
+      assertNotNull(appState.onNodeManagerContainerStarted(instance
+          .getContainerId()));
+    }
+    releaseContainers(completionResults,
+        released,
+        ContainerState.COMPLETE,
+        "released",
+        0
+    );
+  }
+
+  /**
+   * Release a list of containers, updating the completion results.
+   * @param completionResults
+   * @param containerIds
+   * @param containerState
+   * @param exitText
+   * @param containerExitCode
+   * @return
+   */
+  public void releaseContainers(
+      List<AppState.NodeCompletionResult> completionResults,
+      List<ContainerId> containerIds,
+      ContainerState containerState,
+      String exitText,
+      int containerExitCode) {
+    for (ContainerId id : containerIds) {
+      ContainerStatus status = ContainerStatus.newInstance(id,
+          containerState,
+          exitText,
+          containerExitCode);
+      completionResults.add(appState.onCompletedContainer(status));
+    }
+  }
+
+  /**
+   * Create nodes and submit them.
+   * @return a list of roles
+   */
+  public List<RoleInstance> createAndSubmitNodes()
+      throws TriggerClusterTeardownException, SliderInternalStateException {
+    return createAndSubmitNodes(new ArrayList<>());
+  }
+
+  /**
+   * Create nodes and submit them.
+   * @return a list of roles
+   */
+  public List<RoleInstance> createAndSubmitNodes(List<ContainerId> containerIds)
+      throws TriggerClusterTeardownException, SliderInternalStateException {
+    return createAndSubmitNodes(containerIds, new ArrayList<>());
+  }
+
+  /**
+   * Create nodes and submit them.
+   * @return a list of roles allocated
+   */
+  public List<RoleInstance> createAndSubmitNodes(
+      List<ContainerId> containerIds,
+      List<AbstractRMOperation> operationsOut)
+      throws TriggerClusterTeardownException, SliderInternalStateException {
+    List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+    return submitOperations(ops, containerIds, operationsOut);
+  }
+
+  public List<RoleInstance> submitOperations(
+      List<AbstractRMOperation> operationsIn,
+      List<ContainerId> released) {
+    return submitOperations(operationsIn, released, new ArrayList<>());
+  }
+
+  /**
+   * Process the RM operations and send <code>onContainersAllocated</code>
+   * events to the app state.
+   * @param operationsIn list of incoming ops
+   * @param released released containers
+   * @return list of outbound operations
+   */
+  public List<RoleInstance> submitOperations(
+      List<AbstractRMOperation> operationsIn,
+      List<ContainerId> released,
+      List<AbstractRMOperation> operationsOut) {
+    List<Container> allocatedContainers = engine.execute(operationsIn,
+        released);
+    List<ContainerAssignment> assignments = new ArrayList<>();
+    appState.onContainersAllocated(allocatedContainers, assignments,
+        operationsOut);
+
+    List<RoleInstance> roles = new ArrayList<>();
+    for (ContainerAssignment assigned : assignments) {
+      Container container = assigned.container;
+      RoleInstance ri = roleInstance(assigned);
+      //tell the app it arrived
+      LOG.debug("Start submitted {} on ${}", ri.role, container.getId());
+      appState.containerStartSubmitted(container, ri);
+      roles.add(ri);
+    }
+    return roles;
+  }
+
+  /**
+   * Add the AM to the app state.
+   */
+  protected void addAppMastertoAppState() {
+//    appState.buildAppMasterNode(
+//        new MockContainerId(applicationAttemptId, 999999L),
+//        "appmaster",
+//        0,
+//        null);
+  }
+
+  /**
+   * Extract the list of container IDs from the list of role instances.
+   * @param instances instance list
+   * @param role role to look up
+   * @return the list of CIDs
+   */
+  public List<ContainerId> extractContainerIds(
+      List<RoleInstance> instances,
+      String role) {
+    List<ContainerId> ids = new ArrayList<>();
+    for (RoleInstance ri : instances) {
+      if (ri.role.equals(role)) {
+        ids.add(ri.getContainerId());
+      }
+    }
+    return ids;
+  }
+
+  /**
+   * Record a node as failing.
+   * @param node
+   * @param id
+   * @param count
+   * @return the entry
+   */
+  public NodeEntry recordAsFailed(NodeInstance node, int id, int count) {
+    NodeEntry entry = node.getOrCreate(id);
+    for (int i = 1; i <= count; i++) {
+      entry.containerCompleted(
+          false,
+          ContainerOutcome.Failed);
+    }
+    return entry;
+  }
+
+  protected void recordAllFailed(int id, int count, List<NodeInstance> nodes) {
+    for (NodeInstance node : nodes) {
+      recordAsFailed(node, id, count);
+    }
+  }
+
+  /**
+   * Get the container request of an indexed entry. Includes some assertions
+   * for better diagnostics
+   * @param ops operation list
+   * @param index index in the list
+   * @return the request.
+   */
+  public AMRMClient.ContainerRequest getRequest(List<AbstractRMOperation> ops,
+      int index) {
+    assertTrue(index < ops.size());
+    AbstractRMOperation op = ops.get(index);
+    assertTrue(op instanceof ContainerRequestOperation);
+    return ((ContainerRequestOperation) op).getRequest();
+  }
+
+  /**
+   * Get the cancel request of an indexed entry. Includes some assertions for
+   * better diagnostics
+   * @param ops operation list
+   * @param index index in the list
+   * @return the request.
+   */
+  public AMRMClient.ContainerRequest getCancel(List<AbstractRMOperation> ops,
+      int index) {
+    assertTrue(index < ops.size());
+    AbstractRMOperation op = ops.get(index);
+    assertTrue(op instanceof CancelSingleRequest);
+    return ((CancelSingleRequest) op).getRequest();
+  }
+
+  /**
+   * Get the single request of a list of operations; includes the check for
+   * the size.
+   * @param ops operations list of size 1
+   * @return the request within the first ContainerRequestOperation
+   */
+  public AMRMClient.ContainerRequest getSingleRequest(
+      List<AbstractRMOperation> ops) {
+    assertEquals(1, ops.size());
+    return getRequest(ops, 0);
+  }
+
+  /**
+   * Get the single request of a list of operations; includes the check for
+   * the size.
+   * @param ops operations list of size 1
+   * @return the request within the first operation
+   */
+  public AMRMClient.ContainerRequest getSingleCancel(
+      List<AbstractRMOperation> ops) {
+    assertEquals(1, ops.size());
+    return getCancel(ops, 0);
+  }
+
+  /**
+   * Get the single release of a list of operations; includes the check for
+   * the size.
+   * @param ops operations list of size 1
+   * @return the request within the first operation
+   */
+  public ContainerReleaseOperation getSingleRelease(
+      List<AbstractRMOperation> ops) {
+    assertEquals(1, ops.size());
+    AbstractRMOperation op = ops.get(0);
+    assertTrue(op instanceof ContainerReleaseOperation);
+    return (ContainerReleaseOperation) op;
+  }
+
+  /**
+   * Get the node information as a large JSON String.
+   * @return
+   */
+  protected String nodeInformationSnapshotAsString()
+      throws UnsupportedEncodingException, JsonProcessingException {
+    return prettyPrintAsJson(stateAccess.getNodeInformationSnapshot());
+  }
+
+  /**
+   * Scan through all containers and assert that the assignment is AA.
+   * @param index role index
+   */
+  protected void assertAllContainersAA(int index) {
+    for (Entry<String, NodeInstance> nodeMapEntry : cloneNodemap().entrySet()) {
+      String name = nodeMapEntry.getKey();
+      NodeInstance ni = nodeMapEntry.getValue();
+      NodeEntry nodeEntry = ni.get(index);
+      assertTrue("too many instances on node " + name, nodeEntry == null ||
+          nodeEntry.isAntiAffinityConstraintHeld());
+    }
+  }
+
+  /**
+   * Get a snapshot of the nodemap of the application state.
+   * @return a cloned nodemap
+   */
+  protected NodeMap cloneNodemap() {
+    return appState.getRoleHistory().cloneNodemap();
+  }
+
+  /**
+   * Issue a nodes updated event.
+   * @param report report to notify
+   * @return response of AM
+   */
+  protected AppState.NodeUpdatedOutcome updateNodes(NodeReport report) {
+    return appState.onNodesUpdated(Collections.singletonList(report));
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
new file mode 100644
index 0000000..66ae0f9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAM.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+/**
+ * Mock AM.
+ */
+public class MockAM extends SliderAppMaster {
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
new file mode 100644
index 0000000..2fcf054
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockAppState.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.state.AbstractClusterServices;
+import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Extended app state that makes more things public.
+ */
+public class MockAppState extends AppState {
+  public static final int RM_MAX_RAM = 4096;
+  public static final int RM_MAX_CORES = 64;
+
+  private long time = -1;
+
+  public MockAppState(AbstractClusterServices recordFactory) {
+    super(recordFactory, new MetricsAndMonitoring());
+    setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES);
+  }
+
+  /**
+   * Instance with a mock record factory.
+   */
+  public MockAppState() {
+    this(new MockClusterServices());
+  }
+
+  public MockAppState(AppStateBindingInfo bindingInfo)
+      throws BadClusterStateException, IOException, BadConfigException {
+    this();
+    buildInstance(bindingInfo);
+  }
+
+  public Map<String, ProviderRole> getRoleMap() {
+    return super.getRoleMap();
+  }
+
+  /**
+   * Current time. if the <code>time</code> field
+   * is set, that value is returned
+   * @return the current time.
+   */
+  protected long now() {
+    if (time > 0) {
+      return time;
+    }
+    return System.currentTimeMillis();
+  }
+
+  public void setTime(long newTime) {
+    this.time = newTime;
+  }
+
+  public void incTime(long inc) {
+    this.time = this.time + inc;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fe5461/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
new file mode 100644
index 0000000..b509625
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockApplicationAttemptId.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+class MockApplicationAttemptId extends ApplicationAttemptId {
+
+  private ApplicationId applicationId;
+  private int attemptId;
+
+  public MockApplicationAttemptId() {
+  }
+
+  public MockApplicationAttemptId(ApplicationId applicationId, int attemptId) {
+    this.applicationId = applicationId;
+    this.attemptId = attemptId;
+  }
+
+  @Override
+  public ApplicationId getApplicationId() {
+    return applicationId;
+  }
+
+  @Override
+  public void setApplicationId(ApplicationId applicationId) {
+    this.applicationId = applicationId;
+  }
+
+  @Override
+  public int getAttemptId() {
+    return attemptId;
+  }
+
+  @Override
+  public void setAttemptId(int attemptId) {
+    this.attemptId = attemptId;
+  }
+
+  @Override
+  protected void build() {
+
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message