Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D3406200CCC for ; Fri, 21 Jul 2017 20:39:21 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D05D816DB3C; Fri, 21 Jul 2017 18:39:21 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A11E416DB36 for ; Fri, 21 Jul 2017 20:39:19 +0200 (CEST) Received: (qmail 22379 invoked by uid 500); 21 Jul 2017 18:39:05 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 20954 invoked by uid 99); 21 Jul 2017 18:39:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 21 Jul 2017 18:39:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id E9B54F32B0; Fri, 21 Jul 2017 18:39:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Fri, 21 Jul 2017 18:39:32 -0000 Message-Id: <232bed0803454fab91a67066351ebe56@git.apache.org> In-Reply-To: <38802499ba39479db37fcece9e51fff7@git.apache.org> References: <38802499ba39479db37fcece9e51fff7@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [30/52] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy unit tests to yarn native services. Contributed by Billie Rinaldi archived-at: Fri, 21 Jul 2017 18:39:22 -0000 http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java new file mode 100644 index 0000000..b0634bf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.slider.api.resource.Application; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockAppState; +import org.apache.slider.server.appmaster.model.mock.MockRoles; +import org.apache.slider.server.appmaster.state.AppStateBindingInfo; +import org.apache.slider.server.appmaster.state.NodeEntry; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.NodeMap; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Test that app state is rebuilt on a restart. + */ +public class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest + implements MockRoles { + + @Override + public String getTestName() { + return "TestMockAppStateRebuildOnAMRestart"; + } + + @Test + public void testRebuild() throws Throwable { + + int r0 = 1; + int r1 = 2; + int r2 = 3; + getRole0Status().setDesired(r0); + getRole1Status().setDesired(r1); + getRole2Status().setDesired(r2); + List instances = createAndStartNodes(); + + int clusterSize = r0 + r1 + r2; + assertEquals(instances.size(), clusterSize); + + //clone the list + List containers = new ArrayList<>(); + for (RoleInstance ri : instances) { + containers.add(ri.container); + } + NodeMap nodemap = appState.getRoleHistory().cloneNodemap(); + + //and rebuild + + AppStateBindingInfo bindingInfo = buildBindingInfo(); + bindingInfo.application = factory.newApplication(r0, r1, r2) + .name(getTestName()); + bindingInfo.liveContainers = containers; + appState = new MockAppState(bindingInfo); + + assertEquals(appState.getLiveContainers().size(), clusterSize); + + appState.getRoleHistory().dump(); + + //check that the app state direct structures match + List r0live = appState.enumLiveNodesInRole(ROLE0); + List r1live = appState.enumLiveNodesInRole(ROLE1); + List r2live = appState.enumLiveNodesInRole(ROLE2); + + assertEquals(r0, r0live.size()); + assertEquals(r1, r1live.size()); + assertEquals(r2, r2live.size()); + + //now examine the role history + NodeMap newNodemap = appState.getRoleHistory().cloneNodemap(); + + for (NodeInstance nodeInstance : newNodemap.values()) { + String hostname = nodeInstance.hostname; + NodeInstance orig = nodemap.get(hostname); + assertNotNull("Null entry in original nodemap for " + hostname, orig); + + for (int i : Arrays.asList(getRole0Status().getKey(), getRole1Status() + .getKey(), getRole2Status().getKey())) { + assertEquals(nodeInstance.getActiveRoleInstances(i), orig + .getActiveRoleInstances(i)); + NodeEntry origRE = orig.getOrCreate(i); + NodeEntry newRE = nodeInstance.getOrCreate(i); + assertEquals(origRE.getLive(), newRE.getLive()); + assertEquals(0, newRE.getStarting()); + } + } + assertEquals(0, appState.reviewRequestAndReleaseNodes().size()); + + Application application = appState.getClusterStatus(); + // verify the AM restart container count was set + Long restarted = application.getNumberOfRunningContainers(); + assertNotNull(restarted); + //and that the count == 1 master + the region servers + assertEquals(restarted.longValue(), (long)containers.size()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java new file mode 100644 index 0000000..946f1c1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockRoles; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.CancelSingleRequest; +import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation; +import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; +import org.apache.slider.server.appmaster.state.ContainerAssignment; +import org.apache.slider.server.appmaster.state.RoleHistoryUtils; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole; + +/** + * Test that the app state lets you ask for nodes, get a specific host, + * release it and then get that one back again. + */ +public class TestMockAppStateRolePlacement extends BaseMockAppStateTest + implements MockRoles { + + @Override + public String getTestName() { + return "TestMockAppStateRolePlacement"; + } + + + @Test + public void testAllocateReleaseRealloc() throws Throwable { + getRole0Status().setDesired(1); + + List ops = appState.reviewRequestAndReleaseNodes(); + ContainerRequestOperation operation = (ContainerRequestOperation)ops + .get(0); + AMRMClient.ContainerRequest request = operation.getRequest(); + assertTrue(request.getRelaxLocality()); + assertNull(request.getNodes()); + assertNull(request.getRacks()); + assertNotNull(request.getCapability()); + + Container allocated = engine.allocateContainer(request); + List assignments = new ArrayList<>(); + List releaseOperations = new ArrayList<>(); + appState.onContainersAllocated(Arrays.asList((Container)allocated), + assignments, releaseOperations); + // verify the release matches the allocation + assertEquals(releaseOperations.size(), 1); + CancelSingleRequest cancelOp = (CancelSingleRequest)releaseOperations + .get(0); + assertNotNull(cancelOp.getRequest()); + assertNotNull(cancelOp.getRequest().getCapability()); + assertEquals(cancelOp.getRequest().getCapability(), allocated + .getResource()); + // now the assignment + assertEquals(assignments.size(), 1); + ContainerAssignment assigned = assignments.get(0); + Container container = assigned.container; + assertEquals(container.getId(), allocated.getId()); + int roleId = assigned.role.getPriority(); + assertEquals(roleId, extractRole(request.getPriority())); + assertEquals(assigned.role.getName(), ROLE0); + String containerHostname = RoleHistoryUtils.hostnameOf(container); + RoleInstance ri = roleInstance(assigned); + //tell the app it arrived + appState.containerStartSubmitted(container, ri); + assertNotNull(appState.onNodeManagerContainerStarted(container.getId())); + assertEquals(getRole0Status().getRunning(), 1); + ops = appState.reviewRequestAndReleaseNodes(); + assertEquals(ops.size(), 0); + + //now it is surplus + getRole0Status().setDesired(0); + ops = appState.reviewRequestAndReleaseNodes(); + ContainerReleaseOperation release = (ContainerReleaseOperation) ops.get(0); + + assertEquals(release.getContainerId(), container.getId()); + engine.execute(ops); + assertNotNull(appState.onCompletedContainer(containerStatus(container + .getId())).roleInstance); + + //view the world + appState.getRoleHistory().dump(); + + //now ask for a new one + getRole0Status().setDesired(1); + ops = appState.reviewRequestAndReleaseNodes(); + assertEquals(ops.size(), 1); + operation = (ContainerRequestOperation) ops.get(0); + AMRMClient.ContainerRequest request2 = operation.getRequest(); + assertNotNull(request2); + assertEquals(request2.getNodes().get(0), containerHostname); + assertFalse(request2.getRelaxLocality()); + engine.execute(ops); + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java new file mode 100644 index 0000000..d62a91e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockRoles; +import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * Test that if you have >1 role, the right roles are chosen for release. + */ +public class TestMockAppStateRoleRelease extends BaseMockAppStateTest + implements MockRoles { + + @Override + public String getTestName() { + return "TestMockAppStateRoleRelease"; + } + + /** + * Small cluster with multiple containers per node, + * to guarantee many container allocations on each node. + * @return + */ + @Override + public MockYarnEngine createYarnEngine() { + return new MockYarnEngine(4, 4); + } + + @Test + public void testAllocateReleaseRealloc() throws Throwable { + /** + * Allocate to all nodes + */ + getRole0Status().setDesired(6); + getRole1Status().setDesired(5); + getRole2Status().setDesired(4); + List instances = createAndStartNodes(); + assertEquals(instances.size(), 15); + + //now it is surplus + getRole0Status().setDesired(0); + List ops = appState.reviewRequestAndReleaseNodes(); + + List released = new ArrayList<>(); + engine.execute(ops, released); + List ids = extractContainerIds(instances, ROLE0); + for (ContainerId cid : released) { + assertNotNull(appState.onCompletedContainer(containerStatus(cid)) + .roleInstance); + assertTrue(ids.contains(cid)); + } + + //view the world + appState.getRoleHistory().dump(); + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java new file mode 100644 index 0000000..eaf5271 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.api.resource.Resource; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockRoles; +import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; +import org.apache.slider.server.appmaster.state.AppStateBindingInfo; +import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.junit.Test; + +import java.util.Collections; + +/** + * Test that if you have more than one role, the right roles are chosen for + * release. + */ +public class TestMockAppStateUniqueNames extends BaseMockAppStateTest + implements MockRoles { + + @Override + public String getTestName() { + return "TestMockAppStateUniqueNames"; + } + + /** + * Small cluster with multiple containers per node, + * to guarantee many container allocations on each node. + * @return + */ + @Override + public MockYarnEngine createYarnEngine() { + return new MockYarnEngine(4, 4); + } + + @Override + public AppStateBindingInfo buildBindingInfo() { + AppStateBindingInfo bindingInfo = super.buildBindingInfo(); + bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector(); + return bindingInfo; + } + + @Override + public Application buildApplication() { + Application application = super.buildApplication(); + + Component component = new Component().name("group1").numberOfContainers(2L) + .resource(new Resource().memory("1024").cpus(2)) + .uniqueComponentSupport(true); + + application.getComponents().add(component); + return application; + } + + @Test + public void testDynamicFlexDown() throws Throwable { + createAndStartNodes(); + appState.updateComponents(Collections.singletonMap("group1", 0L)); + createAndStartNodes(); + RoleStatus roleStatus = appState.lookupRoleStatus("group11"); + assertEquals(0, roleStatus.getDesired()); + assertEquals(1024L, roleStatus.getResourceRequirements().getMemorySize()); + assertEquals(2, roleStatus.getResourceRequirements().getVirtualCores()); + assertEquals("group1", roleStatus.getGroup()); + } + + @Test + public void testDynamicFlexUp() throws Throwable { + createAndStartNodes(); + appState.updateComponents(Collections.singletonMap("group1", 3L)); + createAndStartNodes(); + RoleStatus group11 = appState.lookupRoleStatus("group11"); + RoleStatus group12 = appState.lookupRoleStatus("group12"); + RoleStatus group13 = appState.lookupRoleStatus("group13"); + assertEquals(1, group11.getDesired()); + assertEquals(1, group12.getDesired()); + assertEquals(1, group13.getDesired()); + assertEquals(1024L, group11.getResourceRequirements().getMemorySize()); + assertEquals(1024L, group12.getResourceRequirements().getMemorySize()); + assertEquals(1024L, group13.getResourceRequirements().getMemorySize()); + assertEquals(2, group11.getResourceRequirements().getVirtualCores()); + assertEquals(2, group12.getResourceRequirements().getVirtualCores()); + assertEquals(2, group13.getResourceRequirements().getVirtualCores()); + assertEquals("group1", group11.getGroup()); + assertEquals("group1", group12.getGroup()); + assertEquals("group1", group13.getGroup()); + + appState.refreshClusterStatus(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java new file mode 100644 index 0000000..046bd83 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Component; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockAppState; +import org.apache.slider.server.appmaster.model.mock.MockRoles; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.operations.ContainerRequestOperation; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; + +/** + * Test the container resource allocation logic. + */ +public class TestMockContainerResourceAllocations extends BaseMockAppStateTest { + + @Override + public Application buildApplication() { + return factory.newApplication(1, 0, 0).name(getTestName()); + } + + @Test + public void testNormalAllocations() throws Throwable { + Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0); + role0.resource(new org.apache.slider.api.resource.Resource().memory("512") + .cpus(2)); + appState.updateComponents(Collections.singletonMap(role0.getName(), + role0.getNumberOfContainers())); + List ops = appState.reviewRequestAndReleaseNodes(); + assertEquals(1, ops.size()); + ContainerRequestOperation operation = (ContainerRequestOperation) ops + .get(0); + Resource requirements = operation.getRequest().getCapability(); + assertEquals(512L, requirements.getMemorySize()); + assertEquals(2, requirements.getVirtualCores()); + } + + @Test + public void testMaxMemAllocations() throws Throwable { + // max core allocations no longer supported + Component role0 = appState.getClusterStatus().getComponent(MockRoles.ROLE0); + role0.resource(new org.apache.slider.api.resource.Resource() + .memory(ResourceKeys.YARN_RESOURCE_MAX).cpus(2)); + appState.updateComponents(Collections.singletonMap(role0.getName(), + role0.getNumberOfContainers())); + List ops = appState.reviewRequestAndReleaseNodes(); + assertEquals(1, ops.size()); + ContainerRequestOperation operation = (ContainerRequestOperation) ops + .get(0); + Resource requirements = operation.getRequest().getCapability(); + assertEquals(MockAppState.RM_MAX_RAM, requirements.getMemorySize()); + assertEquals(2, requirements.getVirtualCores()); + } + + @Test + public void testMaxDefaultAllocations() throws Throwable { + List ops = appState.reviewRequestAndReleaseNodes(); + assertEquals(ops.size(), 1); + ContainerRequestOperation operation = (ContainerRequestOperation) ops + .get(0); + Resource requirements = operation.getRequest().getCapability(); + assertEquals(ResourceKeys.DEF_YARN_MEMORY, requirements.getMemorySize()); + assertEquals(ResourceKeys.DEF_YARN_CORES, requirements.getVirtualCores()); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java new file mode 100644 index 0000000..a3f8abd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.model.mock.MockRoles; +import org.apache.slider.server.appmaster.model.mock.MockYarnEngine; +import org.apache.slider.server.appmaster.operations.AbstractRMOperation; +import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * Test Anti-affine placement. + */ +public class TestMockLabelledAAPlacement extends BaseMockAppStateAATest + implements MockRoles { + private static final Logger LOG = + LoggerFactory.getLogger(TestMockLabelledAAPlacement.class); + + private static final int NODES = 3; + private static final int GPU_NODES = 2; + private static final String HOST0 = "00000000"; + private static final String HOST1 = "00000001"; + + @Override + public void setup() throws Exception { + super.setup(); + + updateNodes(MockFactory.INSTANCE.newNodeReport(HOST0, NodeState.RUNNING, + LABEL_GPU)); + updateNodes(MockFactory.INSTANCE.newNodeReport(HOST1, NodeState.RUNNING, + LABEL_GPU)); + } + + @Override + public MockYarnEngine createYarnEngine() { + return new MockYarnEngine(NODES, 8); + } + + void assertAllContainersAA() { + assertAllContainersAA(getGpuRole().getKey()); + } + + /** + * + * @throws Throwable + */ + @Test + public void testAskForTooMany() throws Throwable { + RoleStatus gpuRole = getGpuRole(); + + describe("Ask for 1 more than the no of available nodes;" + + " expect the final request to be unsatisfied until the cluster " + + "changes size"); + //more than expected + int size = GPU_NODES; + gpuRole.setDesired(size + 1); + + List operations = appState + .reviewRequestAndReleaseNodes(); + assertTrue(gpuRole.isAARequestOutstanding()); + + assertEquals(gpuRole.getAAPending(), size); + for (int i = 0; i < size; i++) { + String iter = "Iteration " + i + " role = " + getAaRole(); + describe(iter); + List operationsOut = new ArrayList<>(); + + List roleInstances = submitOperations(operations, + EMPTY_ID_LIST, operationsOut); + // one instance per request + assertEquals(1, roleInstances.size()); + appState.onNodeManagerContainerStarted(roleInstances.get(0) + .getContainerId()); + assertAllContainersAA(); + // there should be none left + LOG.debug(nodeInformationSnapshotAsString()); + operations = operationsOut; + if (i + 1 < size) { + assertEquals(2, operations.size()); + } else { + assertEquals(1, operations.size()); + } + } + // expect an outstanding AA request to be unsatisfied + assertTrue(gpuRole.getRunning() < gpuRole.getDesired()); + assertEquals(0, gpuRole.getRequested()); + assertFalse(gpuRole.isAARequestOutstanding()); + List allocatedContainers = engine.execute(operations, + EMPTY_ID_LIST); + assertEquals(0, allocatedContainers.size()); + // in a review now, no more requests can be generated, as there is no + // space for AA placements, even though there is cluster capacity + assertEquals(0, appState.reviewRequestAndReleaseNodes().size()); + + // switch node 2 into being labelled + NodeUpdatedOutcome outcome = updateNodes(MockFactory.INSTANCE. + newNodeReport("00000002", NodeState.RUNNING, "gpu")); + + assertEquals(NODES, cloneNodemap().size()); + assertTrue(outcome.clusterChanged); + // no active calls to empty + assertTrue(outcome.operations.isEmpty()); + assertEquals(1, appState.reviewRequestAndReleaseNodes().size()); + } + + protected AppState.NodeUpdatedOutcome addNewNode() { + return updateNodes(MockFactory.INSTANCE.newNodeReport("00000004", + NodeState.RUNNING, "gpu")); + } + + @Test + public void testClusterSizeChangesDuringRequestSequence() throws Throwable { + RoleStatus gpuRole = getGpuRole(); + describe("Change the cluster size where the cluster size changes during " + + "a test sequence."); + gpuRole.setDesired(GPU_NODES + 1); + List operations = appState + .reviewRequestAndReleaseNodes(); + assertTrue(gpuRole.isAARequestOutstanding()); + assertEquals(GPU_NODES, gpuRole.getAAPending()); + NodeUpdatedOutcome outcome = addNewNode(); + assertTrue(outcome.clusterChanged); + // one call to cancel + assertEquals(1, outcome.operations.size()); + // and on a review, one more to rebuild + assertEquals(1, appState.reviewRequestAndReleaseNodes().size()); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java new file mode 100644 index 0000000..5ae626e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; +import org.apache.slider.server.appmaster.state.ContainerPriority; +import org.apache.slider.server.appmaster.state.OutstandingRequest; +import org.apache.slider.utils.SliderTestBase; +import org.junit.Test; + +/** + * Test outstanding request validation. + */ +public class TestOutstandingRequestValidation extends SliderTestBase { + + private static final String[] H1 = hosts("one"); + + @Test + public void testRelaxedNohostsOrLabels() throws Throwable { + createAndValidate(null, null, true); + } + + @Test + public void testRelaxedLabels() throws Throwable { + createAndValidate(null, "gpu", true); + } + + @Test + public void testNonRelaxedLabels() throws Throwable { + expectCreationFailure(null, "gpu", false); + } + + @Test + public void testRelaxedHostNoLabel() throws Throwable { + createAndValidate(H1, "", true); + } + + /** + * Use varargs for simple list to array conversion. + * @param hostnames host names + * @return + */ + public static String[] hosts(String...hostnames) { + return hostnames; + } + + void expectCreationFailure( + String[] hosts, + String labels, + boolean relaxLocality) { + try { + ContainerRequest result = createAndValidate(hosts, labels, relaxLocality); + fail("Expected an exception, got " + result); + } catch (IllegalArgumentException expected) { + assertTrue(expected.toString() + .contains("Can't turn off locality relaxation on a request with no " + + "location constraints")); + } + } + + AMRMClient.ContainerRequest createAndValidate( + String[] hosts, + String labels, + boolean relaxLocality) { + int cores = 1; + int memory = 64; + int p = 1; + Priority pri = ContainerPriority.createPriority(p, !relaxLocality); + ContainerRequest issuedRequest = + newRequest(pri, hosts, labels, relaxLocality); + OutstandingRequest.validateContainerRequest(issuedRequest, p, ""); + return issuedRequest; + } + + AMRMClient.ContainerRequest newRequest( + Priority pri, + String[] hosts, + String labels, + boolean relaxLocality) { + int cores = 1; + int memory = 64; + Resource resource = Resource.newInstance(memory, cores); + return new AMRMClient.ContainerRequest(resource, + hosts, + null, + pri, + relaxLocality, + labels); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java new file mode 100644 index 0000000..077a6d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.types.NodeInformation; +import org.apache.slider.api.types.NodeInformationList; +import org.apache.slider.api.types.RestTypeMarshalling; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; +import org.apache.slider.server.appmaster.state.NodeEntry; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.NodeMap; +import org.apache.slider.server.appmaster.state.RoleHistory; +import org.apache.slider.utils.SliderTestBase; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Test anti-affine placement. + */ +public class TestRoleHistoryAA extends SliderTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(TestRoleHistoryAA.class); + + private List hostnames = Arrays.asList("1", "2", "3"); + private NodeMap nodeMap, gpuNodeMap; + private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + + public TestRoleHistoryAA() throws BadConfigException { + } + + @Override + public void setup() throws Exception { + super.setup(); + nodeMap = createNodeMap(hostnames, NodeState.RUNNING, ""); + gpuNodeMap = createNodeMap(hostnames, NodeState.RUNNING, "GPU"); + } + + @Test + public void testFindNodesInFullCluster() throws Throwable { + // all three will surface at first + verifyResultSize(3, nodeMap.findAllNodesForRole(1, "")); + } + + @Test + public void testFindNodesInUnhealthyCluster() throws Throwable { + // all three will surface at first + markNodeOneUnhealthy(); + verifyResultSize(2, nodeMap.findAllNodesForRole(1, "")); + } + + public boolean markNodeOneUnhealthy() { + return setNodeState(nodeMap.get("1"), NodeState.UNHEALTHY); + } + + protected boolean setNodeState(NodeInstance node, NodeState state) { + return node.updateNode(MockFactory.INSTANCE.newNodeReport(node.hostname, + state, "")); + } + + @Test + public void testFindNoNodesWrongLabel() throws Throwable { + // all three will surface at first + verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU")); + } + + @Test + public void testFindSomeNodesSomeLabel() throws Throwable { + // all three will surface at first + update(nodeMap, + Arrays.asList(MockFactory.INSTANCE.newNodeReport("1", NodeState + .RUNNING, "GPU"))); + List gpuNodes = nodeMap.findAllNodesForRole(1, "GPU"); + verifyResultSize(1, gpuNodes); + NodeInstance instance = gpuNodes.get(0); + instance.getOrCreate(1).onStarting(); + assertFalse(instance.canHost(1, "GPU")); + assertFalse(instance.canHost(1, "")); + verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU")); + + } + + @Test + public void testFindNoNodesRightLabel() throws Throwable { + // all three will surface at first + verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "GPU")); + } + + @Test + public void testFindNoNodesNoLabel() throws Throwable { + // all three will surface at first + verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "")); + } + + @Test + public void testFindNoNodesClusterRequested() throws Throwable { + // all three will surface at first + for (NodeInstance ni : nodeMap.values()) { + ni.getOrCreate(1).request(); + } + assertNoAvailableNodes(1); + } + + @Test + public void testFindNoNodesClusterBusy() throws Throwable { + // all three will surface at first + for (NodeInstance ni : nodeMap.values()) { + ni.getOrCreate(1).request(); + } + assertNoAvailableNodes(1); + } + + /** + * Tag all nodes as starting, then walk one through a bit + * more of its lifecycle. + */ + @Test + public void testFindNoNodesLifecycle() throws Throwable { + // all three will surface at first + for (NodeInstance ni : nodeMap.values()) { + ni.getOrCreate(1).onStarting(); + } + assertNoAvailableNodes(1); + + // walk one of the nodes through the lifecycle + NodeInstance node1 = nodeMap.get("1"); + assertFalse(node1.canHost(1, "")); + node1.get(1).onStartCompleted(); + assertFalse(node1.canHost(1, "")); + assertNoAvailableNodes(1); + node1.get(1).release(); + assertTrue(node1.canHost(1, "")); + List list2 = + verifyResultSize(1, nodeMap.findAllNodesForRole(1, "")); + assertEquals(list2.get(0).hostname, "1"); + + // now tag that node as unhealthy and expect it to go away + markNodeOneUnhealthy(); + assertNoAvailableNodes(1); + } + + @Test + public void testRolesIndependent() throws Throwable { + NodeInstance node1 = nodeMap.get("1"); + NodeEntry role1 = node1.getOrCreate(1); + NodeEntry role2 = node1.getOrCreate(2); + for (NodeInstance ni : nodeMap.values()) { + ni.updateNode(MockFactory.INSTANCE.newNodeReport("0", NodeState + .UNHEALTHY, "")); + } + assertNoAvailableNodes(1); + assertNoAvailableNodes(2); + assertTrue(setNodeState(node1, NodeState.RUNNING)); + // tag role 1 as busy + role1.onStarting(); + assertNoAvailableNodes(1); + + verifyResultSize(1, nodeMap.findAllNodesForRole(2, "")); + assertTrue(node1.canHost(2, "")); + } + + @Test + public void testNodeEntryAvailablity() throws Throwable { + NodeEntry entry = new NodeEntry(1); + assertTrue(entry.isAvailable()); + entry.onStarting(); + assertFalse(entry.isAvailable()); + entry.onStartCompleted(); + assertFalse(entry.isAvailable()); + entry.release(); + assertTrue(entry.isAvailable()); + entry.onStarting(); + assertFalse(entry.isAvailable()); + entry.onStartFailed(); + assertTrue(entry.isAvailable()); + } + + @Test + public void testNodeInstanceSerialization() throws Throwable { + MockRoleHistory rh2 = new MockRoleHistory(new ArrayList<>()); + rh2.getOrCreateNodeInstance("localhost"); + NodeInstance instance = rh2.getOrCreateNodeInstance("localhost"); + instance.getOrCreate(1).onStartCompleted(); + Map naming = Collections.singletonMap(1, "manager"); + NodeInformation ni = instance.serialize(naming); + assertEquals(1, ni.entries.get("manager").live); + NodeInformation ni2 = rh2.getNodeInformation("localhost", naming); + assertEquals(1, ni2.entries.get("manager").live); + Map info = rh2.getNodeInformationSnapshot(naming); + assertEquals(1, info.get("localhost").entries.get("manager").live); + NodeInformationList nil = new NodeInformationList(info.values()); + assertEquals(1, nil.get(0).entries.get("manager").live); + + Messages.NodeInformationProto nodeInformationProto = + RestTypeMarshalling.marshall(ni); + Messages.NodeEntryInformationProto entryProto = nodeInformationProto + .getEntries(0); + assertNotNull(entryProto); + assertEquals(1, entryProto.getPriority()); + NodeInformation unmarshalled = + RestTypeMarshalling.unmarshall(nodeInformationProto); + assertEquals(unmarshalled.hostname, ni.hostname); + assertTrue(unmarshalled.entries.keySet().containsAll(ni.entries.keySet())); + + } + + @Test + public void testBuildRolenames() throws Throwable { + + } + public List assertNoAvailableNodes(int role) { + String label = ""; + return verifyResultSize(0, nodeMap.findAllNodesForRole(role, label)); + } + + List verifyResultSize(int size, List list) { + if (list.size() != size) { + for (NodeInstance ni : list) { + LOG.error(ni.toFullString()); + } + } + assertEquals(size, list.size()); + return list; + } + + NodeMap createNodeMap(List nodeReports) + throws BadConfigException { + NodeMap newNodeMap = new NodeMap(1); + update(newNodeMap, nodeReports); + return newNodeMap; + } + + protected boolean update(NodeMap nm, List nodeReports) { + return nm.buildOrUpdate(nodeReports); + } + + NodeMap createNodeMap(List hosts, NodeState state, + String label) throws BadConfigException { + return createNodeMap(MockFactory.INSTANCE.createNodeReports(hosts, state, + label)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java new file mode 100644 index 0000000..36a480e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.java @@ -0,0 +1,447 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockContainer; +import org.apache.slider.server.appmaster.model.mock.MockNodeId; +import org.apache.slider.server.appmaster.state.ContainerOutcome; +import org.apache.slider.server.appmaster.state.ContainerPriority; +import org.apache.slider.server.appmaster.state.NodeEntry; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.NodeMap; +import org.apache.slider.server.appmaster.state.RoleHistory; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Test container events at the role history level -one below + * the App State. + */ +public class TestRoleHistoryContainerEvents extends BaseMockAppStateTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestRoleHistoryContainerEvents.class); + + @Override + public String getTestName() { + return "TestRoleHistoryContainerEvents"; + } + + private NodeInstance age1Active4; + private NodeInstance age2Active2; + private NodeInstance age3Active0; + private NodeInstance age4Active1; + private NodeInstance age2Active0; + + private RoleHistory roleHistory; + + private Resource resource; + + AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) { + return roleHistory.requestContainerForRole(roleStatus).getIssuedRequest(); + } + + @Override + public void setup() throws Exception { + super.setup(); + + age1Active4 = nodeInstance(1, 4, 0, 0); + age2Active2 = nodeInstance(2, 2, 0, 1); + age3Active0 = nodeInstance(3, 0, 0, 0); + age4Active1 = nodeInstance(4, 1, 0, 0); + age2Active0 = nodeInstance(2, 0, 0, 0); + + roleHistory = appState.getRoleHistory(); + roleHistory.insert(Arrays.asList(age2Active2, age2Active0, + age4Active1, age1Active4, age3Active0)); + roleHistory.buildRecentNodeLists(); + resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES, + ResourceKeys.DEF_YARN_MEMORY); + } + + @Test + public void testFindAndCreate() throws Throwable { + RoleStatus roleStatus = getRole0Status(); + + AMRMClient.ContainerRequest request = + requestContainer(roleStatus); + + List requestNodes = request.getNodes(); + assertNotNull(requestNodes); + assertEquals(1, requestNodes.size()); + String hostname = requestNodes.get(0); + assertEquals(hostname, age3Active0.hostname); + + //build a container + MockContainer container = factory.newContainer(); + container.setNodeId(new MockNodeId(hostname, 0)); + container.setPriority(request.getPriority()); + roleHistory.onContainerAssigned(container); + + NodeMap nodemap = roleHistory.cloneNodemap(); + NodeInstance allocated = nodemap.get(hostname); + NodeEntry roleEntry = allocated.get(roleStatus.getKey()); + assertEquals(1, roleEntry.getStarting()); + assertFalse(roleEntry.isAvailable()); + RoleInstance ri = new RoleInstance(container); + //start it + roleHistory.onContainerStartSubmitted(container, ri); + //later, declare that it started + roleHistory.onContainerStarted(container); + assertEquals(0, roleEntry.getStarting()); + assertFalse(roleEntry.isAvailable()); + assertEquals(1, roleEntry.getActive()); + assertEquals(1, roleEntry.getLive()); + } + + @Test + public void testCreateAndRelease() throws Throwable { + RoleStatus roleStatus = getRole1Status(); + + //verify it is empty + assertTrue(roleHistory.listActiveNodes(roleStatus.getKey()).isEmpty()); + + AMRMClient.ContainerRequest request = + requestContainer(roleStatus); + + assertNull(request.getNodes()); + + //pick an idle host + String hostname = age3Active0.hostname; + + //build a container + MockContainer container = factory.newContainer(new MockNodeId(hostname, + 0), request.getPriority()); + roleHistory.onContainerAssigned(container); + + NodeMap nodemap = roleHistory.cloneNodemap(); + NodeInstance allocated = nodemap.get(hostname); + NodeEntry roleEntry = allocated.get(roleStatus.getKey()); + assertEquals(1, roleEntry.getStarting()); + assertFalse(roleEntry.isAvailable()); + RoleInstance ri = new RoleInstance(container); + //start it + roleHistory.onContainerStartSubmitted(container, ri); + //later, declare that it started + roleHistory.onContainerStarted(container); + assertEquals(0, roleEntry.getStarting()); + assertFalse(roleEntry.isAvailable()); + assertEquals(1, roleEntry.getActive()); + assertEquals(1, roleEntry.getLive()); + + // now pick that instance to destroy + List activeNodes = roleHistory.listActiveNodes(roleStatus + .getKey()); + + + assertEquals(1, activeNodes.size()); + NodeInstance target = activeNodes.get(0); + assertEquals(target, allocated); + roleHistory.onContainerReleaseSubmitted(container); + assertEquals(1, roleEntry.getReleasing()); + assertEquals(1, roleEntry.getLive()); + assertEquals(0, roleEntry.getActive()); + + // release completed + roleHistory.onReleaseCompleted(container); + assertEquals(0, roleEntry.getReleasing()); + assertEquals(0, roleEntry.getLive()); + assertEquals(0, roleEntry.getActive()); + + // verify it is empty + assertTrue(roleHistory.listActiveNodes(roleStatus.getKey()).isEmpty()); + + // ask for a container and expect to get the recently released one + AMRMClient.ContainerRequest request2 = + requestContainer(roleStatus); + + List nodes2 = request2.getNodes(); + assertNotNull(nodes2); + String hostname2 = nodes2.get(0); + + //pick an idle host + assertEquals(hostname2, age3Active0.hostname); + } + + + @Test + public void testStartWithoutWarning() throws Throwable { + //pick an idle host + String hostname = age3Active0.hostname; + //build a container + MockContainer container = factory.newContainer( + new MockNodeId(hostname, 0), + ContainerPriority.createPriority(getRole0Status().getKey(), false)); + + NodeMap nodemap = roleHistory.cloneNodemap(); + NodeInstance allocated = nodemap.get(hostname); + NodeEntry roleEntry = allocated.get(getRole0Status().getKey()); + + //tell RH that it started + roleHistory.onContainerStarted(container); + assertEquals(0, roleEntry.getStarting()); + assertFalse(roleEntry.isAvailable()); + assertEquals(1, roleEntry.getActive()); + assertEquals(1, roleEntry.getLive()); + } + + @Test + public void testStartFailed() throws Throwable { + RoleStatus roleStatus = getRole0Status(); + + AMRMClient.ContainerRequest request = + requestContainer(roleStatus); + + LOG.info("req {}", request); + LOG.info("{}", request.getNodes()); + String hostname = request.getNodes().get(0); + assertEquals(hostname, age3Active0.hostname); + + //build a container + MockContainer container = factory.newContainer(new MockNodeId(hostname, + 0), request.getPriority()); + roleHistory.onContainerAssigned(container); + + NodeMap nodemap = roleHistory.cloneNodemap(); + NodeInstance allocated = nodemap.get(hostname); + NodeEntry roleEntry = allocated.get(roleStatus.getKey()); + assertEquals(1, roleEntry.getStarting()); + assertFalse(roleEntry.isAvailable()); + RoleInstance ri = new RoleInstance(container); + //start it + roleHistory.onContainerStartSubmitted(container, ri); + //later, declare that it failed on startup + assertFalse(roleHistory.onNodeManagerContainerStartFailed(container)); + assertEquals(0, roleEntry.getStarting()); + assertEquals(1, roleEntry.getStartFailed()); + assertEquals(1, roleEntry.getFailed()); + assertTrue(roleEntry.isAvailable()); + assertEquals(0, roleEntry.getActive()); + assertEquals(0, roleEntry.getLive()); + } + + @Test + public void testStartFailedWithoutWarning() throws Throwable { + RoleStatus roleStatus = getRole0Status(); + + AMRMClient.ContainerRequest request = + requestContainer(roleStatus); + + String hostname = request.getNodes().get(0); + assertEquals(hostname, age3Active0.hostname); + + //build a container + MockContainer container = factory.newContainer(); + container.setNodeId(new MockNodeId(hostname, 0)); + container.setPriority(request.getPriority()); + + NodeMap nodemap = roleHistory.cloneNodemap(); + NodeInstance allocated = nodemap.get(hostname); + NodeEntry roleEntry = allocated.get(roleStatus.getKey()); + + assertFalse(roleHistory.onNodeManagerContainerStartFailed(container)); + assertEquals(0, roleEntry.getStarting()); + assertEquals(1, roleEntry.getStartFailed()); + assertEquals(1, roleEntry.getFailed()); + assertTrue(roleEntry.isAvailable()); + assertEquals(0, roleEntry.getActive()); + assertEquals(0, roleEntry.getLive()); + } + + @Test + public void testContainerFailed() throws Throwable { + describe("fail a container without declaring it as starting"); + + RoleStatus roleStatus = getRole0Status(); + + AMRMClient.ContainerRequest request = + requestContainer(roleStatus); + + String hostname = request.getNodes().get(0); + assertEquals(hostname, age3Active0.hostname); + + //build a container + MockContainer container = factory.newContainer(); + container.setNodeId(new MockNodeId(hostname, 0)); + container.setPriority(request.getPriority()); + roleHistory.onContainerAssigned(container); + + NodeMap nodemap = roleHistory.cloneNodemap(); + NodeInstance allocated = nodemap.get(hostname); + NodeEntry roleEntry = allocated.get(roleStatus.getKey()); + assertEquals(1, roleEntry.getStarting()); + assertFalse(roleEntry.isAvailable()); + RoleInstance ri = new RoleInstance(container); + //start it + roleHistory.onContainerStartSubmitted(container, ri); + roleHistory.onContainerStarted(container); + + //later, declare that it failed + roleHistory.onFailedContainer( + container, + false, + ContainerOutcome.Failed); + assertEquals(0, roleEntry.getStarting()); + assertTrue(roleEntry.isAvailable()); + assertEquals(0, roleEntry.getActive()); + assertEquals(0, roleEntry.getLive()); + } + + @Test + public void testContainerFailedWithoutWarning() throws Throwable { + describe("fail a container without declaring it as starting"); + RoleStatus roleStatus = getRole0Status(); + + AMRMClient.ContainerRequest request = + requestContainer(roleStatus); + + String hostname = request.getNodes().get(0); + assertEquals(hostname, age3Active0.hostname); + + //build a container + MockContainer container = factory.newContainer(); + container.setNodeId(new MockNodeId(hostname, 0)); + container.setPriority(request.getPriority()); + + + NodeMap nodemap = roleHistory.cloneNodemap(); + NodeInstance allocated = nodemap.get(hostname); + NodeEntry roleEntry = allocated.get(roleStatus.getKey()); + assertTrue(roleEntry.isAvailable()); + roleHistory.onFailedContainer( + container, + false, + ContainerOutcome.Failed); + assertEquals(0, roleEntry.getStarting()); + assertEquals(1, roleEntry.getFailed()); + assertTrue(roleEntry.isAvailable()); + assertEquals(0, roleEntry.getActive()); + assertEquals(0, roleEntry.getLive()); + } + + @Test + public void testAllocationListPrep() throws Throwable { + describe("test prepareAllocationList"); + RoleStatus roleStatus = getRole0Status(); + + AMRMClient.ContainerRequest request = + requestContainer(roleStatus); + + String hostname = request.getNodes().get(0); + assertEquals(hostname, age3Active0.hostname); + + MockContainer container1 = factory.newContainer(); + container1.setNodeId(new MockNodeId(hostname, 0)); + container1.setPriority(Priority.newInstance(getRole0Status().getKey())); + + MockContainer container2 = factory.newContainer(); + container2.setNodeId(new MockNodeId(hostname, 0)); + container2.setPriority(Priority.newInstance(getRole1Status().getKey())); + + // put containers in List with role == 1 first + List containers = Arrays.asList((Container) container2, + (Container) container1); + List sortedContainers = roleHistory.prepareAllocationList( + containers); + + // verify that the first container has role == 0 after sorting + MockContainer c1 = (MockContainer) sortedContainers.get(0); + assertEquals(getRole0Status().getKey(), c1.getPriority().getPriority()); + MockContainer c2 = (MockContainer) sortedContainers.get(1); + assertEquals(getRole1Status().getKey(), c2.getPriority().getPriority()); + } + + @Test + public void testNodeUpdated() throws Throwable { + describe("fail a node"); + + RoleStatus roleStatus = getRole0Status(); + + AMRMClient.ContainerRequest request = + roleHistory.requestContainerForRole(roleStatus).getIssuedRequest(); + + String hostname = request.getNodes().get(0); + assertEquals(age3Active0.hostname, hostname); + + // build a container + MockContainer container = factory.newContainer(new MockNodeId(hostname, + 0), request.getPriority()); + + roleHistory.onContainerAssigned(container); + + NodeMap nodemap = roleHistory.cloneNodemap(); + NodeInstance allocated = nodemap.get(hostname); + NodeEntry roleEntry = allocated.get(roleStatus.getKey()); + assertEquals(1, roleEntry.getStarting()); + assertFalse(roleEntry.isAvailable()); + RoleInstance ri = new RoleInstance(container); + // start it + roleHistory.onContainerStartSubmitted(container, ri); + roleHistory.onContainerStarted(container); + + int startSize = nodemap.size(); + + // now send a list of updated (failed) nodes event + List nodesUpdated = new ArrayList<>(); + NodeReport nodeReport = NodeReport.newInstance( + NodeId.newInstance(hostname, 0), + NodeState.LOST, + null, null, null, null, 1, null, 0); + nodesUpdated.add(nodeReport); + roleHistory.onNodesUpdated(nodesUpdated); + + nodemap = roleHistory.cloneNodemap(); + int endSize = nodemap.size(); + // as even unused nodes are added to the list, we expect the map size to + // be >1 + assertTrue(startSize <= endSize); + assertNotNull(nodemap.get(hostname)); + assertFalse(nodemap.get(hostname).isOnline()); + + // add a failure of a node we've never head of + String newhost = "newhost"; + nodesUpdated = Arrays.asList( + NodeReport.newInstance( + NodeId.newInstance(newhost, 0), + NodeState.LOST, + null, null, null, null, 1, null, 0) + ); + roleHistory.onNodesUpdated(nodesUpdated); + + NodeMap nodemap2 = roleHistory.cloneNodemap(); + assertNotNull(nodemap2.get(newhost)); + assertFalse(nodemap2.get(newhost).isOnline()); + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java new file mode 100644 index 0000000..2d49e26 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.model.mock.MockRoleHistory; +import org.apache.slider.server.appmaster.state.ContainerOutcome; +import org.apache.slider.server.appmaster.state.NodeEntry; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.RoleHistory; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Testing finding nodes for new instances. + * + * This stresses the non-AA codepath + */ +public class TestRoleHistoryFindNodesForNewInstances extends + BaseMockAppStateTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestRoleHistoryFindNodesForNewInstances.class); + + public TestRoleHistoryFindNodesForNewInstances() throws BadConfigException { + } + + @Override + public String getTestName() { + return "TestFindNodesForNewInstances"; + } + + private NodeInstance age1Active4; + private NodeInstance age2Active2; + private NodeInstance age3Active0; + private NodeInstance age4Active1; + private NodeInstance age2Active0; + + private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES); + + private RoleStatus roleStat; + private RoleStatus roleStat2; + + @Override + public void setup() throws Exception { + super.setup(); + + age1Active4 = nodeInstance(1, 4, 0, 0); + age2Active2 = nodeInstance(2, 2, 0, 1); + age3Active0 = nodeInstance(3, 0, 0, 0); + age4Active1 = nodeInstance(4, 1, 0, 0); + age2Active0 = nodeInstance(2, 0, 0, 0); + + roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1, + age1Active4, age3Active0)); + roleHistory.buildRecentNodeLists(); + + roleStat = getRole0Status(); + roleStat2 = getRole2Status(); + } + + public List findNodes(int count) { + return findNodes(count, roleStat); + } + + public List findNodes(int count, RoleStatus roleStatus) { + List found = new ArrayList<>(); + for (int i = 0; i < count; i++) { + NodeInstance f = roleHistory.findRecentNodeForNewInstance(roleStatus); + if (f != null) { + found.add(f); + } + } + return found; + } + + @Test + public void testFind1NodeR0() throws Throwable { + NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat); + LOG.info("found: {}", found); + assertTrue(Arrays.asList(age3Active0).contains(found)); + } + + @Test + public void testFind2NodeR0() throws Throwable { + NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat); + LOG.info("found: {}", found); + assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found)); + NodeInstance found2 = roleHistory.findRecentNodeForNewInstance(roleStat); + LOG.info("found: {}", found2); + assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found2)); + assertNotEquals(found, found2); + } + + @Test + public void testFind3NodeR0ReturnsNull() throws Throwable { + assertEquals(2, findNodes(2).size()); + NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat); + assertNull(found); + } + + @Test + public void testFindNodesOneEntry() throws Throwable { + List foundNodes = findNodes(4, roleStat2); + assertEquals(0, foundNodes.size()); + } + + @Test + public void testFindNodesIndependent() throws Throwable { + assertEquals(2, findNodes(2).size()); + roleHistory.dump(); + assertEquals(0, findNodes(3, roleStat2).size()); + } + + @Test + public void testFindNodesFallsBackWhenUsed() throws Throwable { + // mark age2 and active 0 as busy, expect a null back + age2Active0.get(getRole0Status().getKey()).onStartCompleted(); + assertNotEquals(0, age2Active0.getActiveRoleInstances(getRole0Status() + .getKey())); + age3Active0.get(getRole0Status().getKey()).onStartCompleted(); + assertNotEquals(0, age3Active0.getActiveRoleInstances(getRole0Status() + .getKey())); + NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat); + if (found != null) { + LOG.info(found.toFullString()); + } + assertNull(found); + } + @Test + public void testFindNodesSkipsFailingNode() throws Throwable { + // mark age2 and active 0 as busy, expect a null back + + NodeEntry entry0 = age2Active0.get(getRole0Status().getKey()); + entry0.containerCompleted( + false, + ContainerOutcome.Failed); + assertTrue(entry0.getFailed() > 0); + assertTrue(entry0.getFailedRecently() > 0); + entry0.containerCompleted( + false, + ContainerOutcome.Failed); + assertFalse(age2Active0.exceedsFailureThreshold(roleStat)); + // set failure to 1 + roleStat.getProviderRole().nodeFailureThreshold = 1; + // threshold is now exceeded + assertTrue(age2Active0.exceedsFailureThreshold(roleStat)); + + // get the role & expect age3 to be picked up, even though it is older + NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat); + assertEquals(age3Active0, found); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/751ce72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java new file mode 100644 index 0000000..91abaa4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.history; + +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockFactory; +import org.apache.slider.server.appmaster.state.NodeInstance; +import org.apache.slider.server.appmaster.state.RoleStatus; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Unit test to verify the comparators sort as expected. + */ +public class TestRoleHistoryNIComparators extends BaseMockAppStateTest { + + private NodeInstance age1Active4; + private NodeInstance age2Active2; + private NodeInstance age3Active0; + private NodeInstance age4Active1; + private NodeInstance empty = new NodeInstance("empty", MockFactory + .ROLE_COUNT); + private NodeInstance age6failing; + private NodeInstance age1failing; + + private List nodes; + private List nodesPlusEmpty; + private List allnodes; + + private RoleStatus role0Status; + + @Override + public void setup() throws Exception { + super.setup(); + + role0Status = getRole0Status(); + + age1Active4 = nodeInstance(1001, 4, 0, 0); + age2Active2 = nodeInstance(1002, 2, 0, 0); + age3Active0 = nodeInstance(1003, 0, 0, 0); + age4Active1 = nodeInstance(1004, 1, 0, 0); + age6failing = nodeInstance(1006, 0, 0, 0); + age1failing = nodeInstance(1001, 0, 0, 0); + + age6failing.get(role0Status.getKey()).setFailedRecently(2); + age1failing.get(role0Status.getKey()).setFailedRecently(1); + + nodes = Arrays.asList(age2Active2, age4Active1, age1Active4, age3Active0); + nodesPlusEmpty = Arrays.asList(age2Active2, age4Active1, age1Active4, + age3Active0, empty); + allnodes = Arrays.asList(age6failing, age2Active2, age4Active1, + age1Active4, age3Active0, age1failing); + } + + @Override + public String getTestName() { + return "TestNIComparators"; + } + + @Test + public void testPreferred() throws Throwable { + Collections.sort(nodes, new NodeInstance.Preferred(role0Status.getKey())); + assertListEquals(nodes, Arrays.asList(age4Active1, age3Active0, + age2Active2, age1Active4)); + } + + /** + * The preferred sort still includes failures; up to next phase in process + * to handle that. + * @throws Throwable + */ + @Test + public void testPreferredWithFailures() throws Throwable { + Collections.sort(allnodes, new NodeInstance.Preferred(role0Status + .getKey())); + assertEquals(allnodes.get(0), age6failing); + assertEquals(allnodes.get(1), age4Active1); + } + + @Test + public void testPreferredComparatorDowngradesFailures() throws Throwable { + NodeInstance.Preferred preferred = new NodeInstance.Preferred(role0Status + .getKey()); + assertEquals(-1, preferred.compare(age6failing, age1failing)); + assertEquals(1, preferred.compare(age1failing, age6failing)); + } + + @Test + public void testNewerThanNoRole() throws Throwable { + Collections.sort(nodesPlusEmpty, new NodeInstance.Preferred(role0Status + .getKey())); + assertListEquals(nodesPlusEmpty, Arrays.asList(age4Active1, age3Active0, + age2Active2, age1Active4, empty)); + } + + @Test + public void testMoreActiveThan() throws Throwable { + + Collections.sort(nodes, new NodeInstance.MoreActiveThan(role0Status + .getKey())); + assertListEquals(nodes, Arrays.asList(age1Active4, age2Active2, + age4Active1, age3Active0)); + } + + @Test + public void testMoreActiveThanEmpty() throws Throwable { + + Collections.sort(nodesPlusEmpty, new NodeInstance.MoreActiveThan( + role0Status.getKey())); + assertListEquals(nodesPlusEmpty, Arrays.asList(age1Active4, age2Active2, + age4Active1, age3Active0, empty)); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org