Return-Path: X-Original-To: apmail-cloudstack-commits-archive@www.apache.org Delivered-To: apmail-cloudstack-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2822F17362 for ; Tue, 14 Oct 2014 12:19:46 +0000 (UTC) Received: (qmail 32868 invoked by uid 500); 14 Oct 2014 12:19:46 -0000 Delivered-To: apmail-cloudstack-commits-archive@cloudstack.apache.org Received: (qmail 32835 invoked by uid 500); 14 Oct 2014 12:19:45 -0000 Mailing-List: contact commits-help@cloudstack.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cloudstack.apache.org Delivered-To: mailing list commits@cloudstack.apache.org Received: (qmail 32825 invoked by uid 99); 14 Oct 2014 12:19:45 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 14 Oct 2014 12:19:45 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 6B35E92D288; Tue, 14 Oct 2014 12:19:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sanjaytripathi@apache.org To: commits@cloudstack.apache.org Message-Id: <6e2af9a9d3264ce993e18328d4075f39@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: updated refs/heads/master to e4a8bbc Date: Tue, 14 Oct 2014 12:19:44 +0000 (UTC) Repository: cloudstack Updated Branches: refs/heads/master 0a9742f91 -> e4a8bbcdf CLOUDSTACK-6650: Reorder Cluster list in deployment planner to protect GPU enabled hosts from non-GPU VM deployment. Cluster reordering is based on the number of unique host tags in a cluster, cluster with most number of unique host tags will put at the end of list. Hosts with GPU capability will get tagged with implicit tags defined by global config param 'implicit.host.tags' at the time os host discovery. Also added FirstFitPlannerTest unit test file. Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/e4a8bbcd Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/e4a8bbcd Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/e4a8bbcd Branch: refs/heads/master Commit: e4a8bbcdf42bd3133e5cb7e3230b45c0e7018426 Parents: 0a9742f Author: Sanjay Tripathi Authored: Tue Oct 14 13:43:35 2014 +0530 Committer: Sanjay Tripathi Committed: Tue Oct 14 17:53:50 2014 +0530 ---------------------------------------------------------------------- .../cloud/agent/api/StartupRoutingCommand.java | 11 + .../src/com/cloud/host/dao/HostTagsDao.java | 2 + .../src/com/cloud/host/dao/HostTagsDaoImpl.java | 17 + .../implicitplanner/ImplicitPlannerTest.java | 12 + .../resource/XenServer620SP1Resource.java | 3 + server/src/com/cloud/configuration/Config.java | 8 + .../src/com/cloud/deploy/FirstFitPlanner.java | 55 ++- .../com/cloud/resource/ResourceManagerImpl.java | 16 + .../vm/DeploymentPlanningManagerImplTest.java | 12 + .../test/com/cloud/vm/FirstFitPlannerTest.java | 387 +++++++++++++++++++ setup/db/db/schema-441to450.sql | 2 + 11 files changed, 521 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/core/src/com/cloud/agent/api/StartupRoutingCommand.java ---------------------------------------------------------------------- diff --git a/core/src/com/cloud/agent/api/StartupRoutingCommand.java b/core/src/com/cloud/agent/api/StartupRoutingCommand.java index 834d076..c413836 100755 --- a/core/src/com/cloud/agent/api/StartupRoutingCommand.java +++ b/core/src/com/cloud/agent/api/StartupRoutingCommand.java @@ -19,7 +19,9 @@ package com.cloud.agent.api; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import com.cloud.host.Host; @@ -39,6 +41,7 @@ public class StartupRoutingCommand extends StartupCommand { String pool; HypervisorType hypervisorType; Map hostDetails; //stuff like host os, cpu capabilities + List hostTags = new ArrayList(); String hypervisorVersion; HashMap> groupDetails = new HashMap>(); @@ -162,6 +165,14 @@ public class StartupRoutingCommand extends StartupCommand { this.hypervisorVersion = hypervisorVersion; } + public List getHostTags() { + return hostTags; + } + + public void setHostTags(String hostTag) { + this.hostTags.add(hostTag); + } + public HashMap> getGpuGroupDetails() { return groupDetails; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/engine/schema/src/com/cloud/host/dao/HostTagsDao.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/com/cloud/host/dao/HostTagsDao.java index 50c0b7e..8901716 100644 --- a/engine/schema/src/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/com/cloud/host/dao/HostTagsDao.java @@ -27,4 +27,6 @@ public interface HostTagsDao extends GenericDao { List gethostTags(long hostId); + List getDistinctImplicitHostTags(List hostIds, String[] implicitHostTags); + } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/engine/schema/src/com/cloud/host/dao/HostTagsDaoImpl.java ---------------------------------------------------------------------- diff --git a/engine/schema/src/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/com/cloud/host/dao/HostTagsDaoImpl.java index 041a8fe..7e8615a 100644 --- a/engine/schema/src/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/com/cloud/host/dao/HostTagsDaoImpl.java @@ -25,19 +25,28 @@ import org.springframework.stereotype.Component; import com.cloud.host.HostTagVO; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.db.SearchCriteria.Func; @Component @Local(value = HostTagsDao.class) public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao { protected final SearchBuilder HostSearch; + protected final GenericSearchBuilder DistinctImplictTagsSearch; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ); HostSearch.done(); + + DistinctImplictTagsSearch = createSearchBuilder(String.class); + DistinctImplictTagsSearch.select(null, Func.DISTINCT, DistinctImplictTagsSearch.entity().getTag()); + DistinctImplictTagsSearch.and("hostIds", DistinctImplictTagsSearch.entity().getHostId(), SearchCriteria.Op.IN); + DistinctImplictTagsSearch.and("implicitTags", DistinctImplictTagsSearch.entity().getTag(), SearchCriteria.Op.IN); + DistinctImplictTagsSearch.done(); } @Override @@ -55,6 +64,14 @@ public class HostTagsDaoImpl extends GenericDaoBase implements } @Override + public List getDistinctImplicitHostTags(List hostIds, String[] implicitHostTags) { + SearchCriteria sc = DistinctImplictTagsSearch.create(); + sc.setParameters("hostIds", hostIds.toArray(new Object[hostIds.size()])); + sc.setParameters("implicitTags", (Object[])implicitHostTags); + return customSearch(sc, null); + } + + @Override public void persist(long hostId, List hostTags) { TransactionLegacy txn = TransactionLegacy.currentTxn(); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/plugins/deployment-planners/implicit-dedication/test/org/apache/cloudstack/implicitplanner/ImplicitPlannerTest.java ---------------------------------------------------------------------- diff --git a/plugins/deployment-planners/implicit-dedication/test/org/apache/cloudstack/implicitplanner/ImplicitPlannerTest.java b/plugins/deployment-planners/implicit-dedication/test/org/apache/cloudstack/implicitplanner/ImplicitPlannerTest.java index 4182193..e834248 100644 --- a/plugins/deployment-planners/implicit-dedication/test/org/apache/cloudstack/implicitplanner/ImplicitPlannerTest.java +++ b/plugins/deployment-planners/implicit-dedication/test/org/apache/cloudstack/implicitplanner/ImplicitPlannerTest.java @@ -69,8 +69,10 @@ import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.deploy.ImplicitDedicationPlanner; import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.gpu.dao.HostGpuGroupsDao; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostTagsDao; import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -468,6 +470,16 @@ public class ImplicitPlannerTest { } @Bean + public HostTagsDao hostTagsDao() { + return Mockito.mock(HostTagsDao.class); + } + + @Bean + public HostGpuGroupsDao hostGpuGroupsDao() { + return Mockito.mock(HostGpuGroupsDao.class); + } + + @Bean public DataCenterDao dcDao() { return Mockito.mock(DataCenterDao.class); } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServer620SP1Resource.java ---------------------------------------------------------------------- diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServer620SP1Resource.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServer620SP1Resource.java index 88e4600..5553553 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServer620SP1Resource.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServer620SP1Resource.java @@ -84,6 +84,9 @@ public class XenServer620SP1Resource extends XenServer620Resource { try { HashMap> groupDetails = getGPUGroupDetails(conn); cmd.setGpuGroupDetails(groupDetails); + if (groupDetails != null && !groupDetails.isEmpty()) { + cmd.setHostTags("GPU"); + } } catch (Exception e) { if (s_logger.isDebugEnabled()) { s_logger.debug("Error while getting GPU device info from host " + cmd.getName(), e); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/server/src/com/cloud/configuration/Config.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 83badbc..a462348 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -1360,6 +1360,14 @@ public enum Config { "false", "Deploys a VM per zone to manage secondary storage if true, otherwise secondary storage is mounted on management server", null), + ImplicitHostTags( + "Hidden", + ManagementServer.class, + String.class, + "implicit.host.tags", + "GPU", + "Tag hosts at the time of host disovery based on the host properties/capabilities", + null), CreatePoolsInPod( "Hidden", ManagementServer.class, http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/server/src/com/cloud/deploy/FirstFitPlanner.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/deploy/FirstFitPlanner.java b/server/src/com/cloud/deploy/FirstFitPlanner.java index 48780a7..8fa25bd 100755 --- a/server/src/com/cloud/deploy/FirstFitPlanner.java +++ b/server/src/com/cloud/deploy/FirstFitPlanner.java @@ -17,6 +17,8 @@ package com.cloud.deploy; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,9 +46,14 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.gpu.GPU; +import com.cloud.gpu.dao.HostGpuGroupsDao; +import com.cloud.host.Host; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostTagsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.offering.ServiceOffering; +import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.StorageManager; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; @@ -102,9 +109,16 @@ public class FirstFitPlanner extends PlannerBase implements DeploymentClusterPla DataStoreManager dataStoreMgr; @Inject protected ClusterDetailsDao _clusterDetailsDao; + @Inject + protected ServiceOfferingDetailsDao _serviceOfferingDetailsDao; + @Inject + protected HostGpuGroupsDao _hostGpuGroupsDao; + @Inject + protected HostTagsDao _hostTagsDao; protected String _allocationAlgorithm = "random"; protected String _globalDeploymentPlanner = "FirstFitPlanner"; + protected String[] _implicitHostTags; @Override public List orderClusters(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid) throws InsufficientServerCapacityException { @@ -131,7 +145,6 @@ public class FirstFitPlanner extends PlannerBase implements DeploymentClusterPla clusterList.add(clusterIdSpecified); removeClustersCrossingThreshold(clusterList, avoid, vmProfile, plan); } - return clusterList; } else { s_logger.debug("The specified cluster cannot be found, returning."); avoid.addCluster(plan.getClusterId()); @@ -152,7 +165,6 @@ public class FirstFitPlanner extends PlannerBase implements DeploymentClusterPla avoid.addPod(plan.getPodId()); } } - return clusterList; } else { s_logger.debug("The specified Pod cannot be found, returning."); avoid.addPod(plan.getPodId()); @@ -164,13 +176,44 @@ public class FirstFitPlanner extends PlannerBase implements DeploymentClusterPla boolean applyAllocationAtPods = Boolean.parseBoolean(_configDao.getValue(Config.ApplyAllocationAlgorithmToPods.key())); if (applyAllocationAtPods) { //start scan at all pods under this zone. - return scanPodsForDestination(vmProfile, plan, avoid); + clusterList = scanPodsForDestination(vmProfile, plan, avoid); } else { //start scan at clusters under this zone. - return scanClustersForDestinationInZoneOrPod(plan.getDataCenterId(), true, vmProfile, plan, avoid); + clusterList = scanClustersForDestinationInZoneOrPod(plan.getDataCenterId(), true, vmProfile, plan, avoid); } } + if (clusterList != null && !clusterList.isEmpty()) { + ServiceOffering offering = vmProfile.getServiceOffering(); + // In case of non-GPU VMs, protect GPU enabled Hosts and prefer VM deployment on non-GPU Hosts. + if ((_serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) == null) && !(_hostGpuGroupsDao.listHostIds().isEmpty())) { + int requiredCpu = offering.getCpu() * offering.getSpeed(); + long requiredRam = offering.getRamSize() * 1024L * 1024L; + reorderClustersBasedOnImplicitTags(clusterList, requiredCpu, requiredRam); + } + } + return clusterList; + } + + private void reorderClustersBasedOnImplicitTags(List clusterList, int requiredCpu, long requiredRam) { + final HashMap UniqueTagsInClusterMap = new HashMap(); + Long uniqueTags; + for (Long clusterId : clusterList) { + uniqueTags = (long) 0; + List hostList = _capacityDao.listHostsWithEnoughCapacity(requiredCpu, requiredRam, clusterId, Host.Type.Routing.toString()); + if (!hostList.isEmpty() && _implicitHostTags.length > 0) { + uniqueTags = new Long(_hostTagsDao.getDistinctImplicitHostTags(hostList, _implicitHostTags).size()); + } + UniqueTagsInClusterMap.put(clusterId, uniqueTags); + } + Collections.sort(clusterList, new Comparator() { + @Override + public int compare(Long o1, Long o2) { + Long t1 = UniqueTagsInClusterMap.get(o1); + Long t2 = UniqueTagsInClusterMap.get(o2); + return t1.compareTo(t2); + } + }); } private List scanPodsForDestination(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid) { @@ -504,6 +547,10 @@ public class FirstFitPlanner extends PlannerBase implements DeploymentClusterPla super.configure(name, params); _allocationAlgorithm = _configDao.getValue(Config.VmAllocationAlgorithm.key()); _globalDeploymentPlanner = _configDao.getValue(Config.VmDeploymentPlanner.key()); + String configValue; + if ((configValue = _configDao.getValue(Config.ImplicitHostTags.key())) != null) { + _implicitHostTags = configValue.trim().split("\\s*,\\s*"); + } return true; } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/server/src/com/cloud/resource/ResourceManagerImpl.java ---------------------------------------------------------------------- diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 477b08f..c7ba80a 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -1632,6 +1632,22 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } + if (startup instanceof StartupRoutingCommand) { + StartupRoutingCommand ssCmd = ((StartupRoutingCommand)startup); + List implicitHostTags = ssCmd.getHostTags(); + if (!implicitHostTags.isEmpty()) { + if (hostTags == null) { + hostTags = _hostTagsDao.gethostTags(host.getId()); + } + if (hostTags != null) { + implicitHostTags.removeAll(hostTags); + hostTags.addAll(implicitHostTags); + } else { + hostTags = implicitHostTags; + } + } + } + host.setDataCenterId(dc.getId()); host.setPodId(podId); host.setClusterId(clusterId); http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java ---------------------------------------------------------------------- diff --git a/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java b/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java index 94c1d18..b9e2831 100644 --- a/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java +++ b/server/test/com/cloud/vm/DeploymentPlanningManagerImplTest.java @@ -76,7 +76,9 @@ import com.cloud.deploy.PlannerHostReservationVO; import com.cloud.deploy.dao.PlannerHostReservationDao; import com.cloud.exception.AffinityConflictException; import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.gpu.dao.HostGpuGroupsDao; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostTagsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; @@ -240,6 +242,11 @@ public class DeploymentPlanningManagerImplTest { } @Bean + public HostTagsDao hostTagsDao() { + return Mockito.mock(HostTagsDao.class); + } + + @Bean public ClusterDetailsDao clusterDetailsDao() { return Mockito.mock(ClusterDetailsDao.class); } @@ -389,6 +396,11 @@ public class DeploymentPlanningManagerImplTest { return Mockito.mock(AffinityGroupService.class); } + @Bean + public HostGpuGroupsDao hostGpuGroupsDap() { + return Mockito.mock(HostGpuGroupsDao.class); + } + public static class Library implements TypeFilter { @Override http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/server/test/com/cloud/vm/FirstFitPlannerTest.java ---------------------------------------------------------------------- diff --git a/server/test/com/cloud/vm/FirstFitPlannerTest.java b/server/test/com/cloud/vm/FirstFitPlannerTest.java new file mode 100644 index 0000000..b7b103b --- /dev/null +++ b/server/test/com/cloud/vm/FirstFitPlannerTest.java @@ -0,0 +1,387 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.test.utils.SpringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Matchers; +import org.mockito.Mockito; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.ComponentScan.Filter; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityManager; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.configuration.Config; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; +import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.deploy.FirstFitPlanner; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.gpu.dao.HostGpuGroupsDao; +import com.cloud.host.Host; +import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostTagsDao; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.StorageManager; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ComponentContext; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; + +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration(loader = AnnotationConfigContextLoader.class) +public class FirstFitPlannerTest { + + @Inject + FirstFitPlanner planner = new FirstFitPlanner(); + @Inject + DataCenterDao dcDao; + @Inject + ClusterDao clusterDao; + @Inject + UserVmDao vmDao; + @Inject + ConfigurationDao configDao; + @Inject + CapacityDao capacityDao; + @Inject + AccountManager accountMgr; + @Inject + ServiceOfferingDao serviceOfferingDao; + @Inject + ServiceOfferingDetailsDao serviceOfferingDetailsDao; + @Inject + HostGpuGroupsDao hostGpuGroupsDao; + @Inject + HostTagsDao hostTagsDao; + + private static long domainId = 1L; + long dataCenterId = 1L; + long accountId = 1L; + long offeringId = 12L; + int noOfCpusInOffering = 1; + int cpuSpeedInOffering = 500; + int ramInOffering = 512; + AccountVO acct = new AccountVO(accountId); + + @Before + public void setUp() { + when(configDao.getValue(Mockito.anyString())).thenReturn(null); + when(configDao.getValue(Config.ImplicitHostTags.key())).thenReturn("GPU"); + + ComponentContext.initComponentsLifeCycle(); + acct.setType(Account.ACCOUNT_TYPE_ADMIN); + acct.setAccountName("user1"); + acct.setDomainId(domainId); + acct.setId(accountId); + } + + @After + public void tearDown() { + CallContext.unregister(); + } + + @Test + public void checkClusterReorderingBasedOnImplicitHostTags() throws InsufficientServerCapacityException { + VirtualMachineProfileImpl vmProfile = mock(VirtualMachineProfileImpl.class); + DataCenterDeployment plan = mock(DataCenterDeployment.class); + ExcludeList avoids = mock(ExcludeList.class); + initializeForTest(vmProfile, plan, avoids); + + List clusterList = planner.orderClusters(vmProfile, plan, avoids); + List reorderedClusterList = new ArrayList(); + reorderedClusterList.add(4L); + reorderedClusterList.add(3L); + reorderedClusterList.add(1L); + reorderedClusterList.add(5L); + reorderedClusterList.add(6L); + reorderedClusterList.add(2L); + + assertTrue("Reordered cluster list is not ownering the implict host tags", (clusterList.equals(reorderedClusterList))); + } + + private void initializeForTest(VirtualMachineProfileImpl vmProfile, DataCenterDeployment plan, ExcludeList avoids) { + DataCenterVO mockDc = mock(DataCenterVO.class); + VMInstanceVO vm = mock(VMInstanceVO.class); + UserVmVO userVm = mock(UserVmVO.class); + ServiceOfferingVO offering = mock(ServiceOfferingVO.class); + + AccountVO account = mock(AccountVO.class); + when(account.getId()).thenReturn(accountId); + when(account.getAccountId()).thenReturn(accountId); + when(vmProfile.getOwner()).thenReturn(account); + when(vmProfile.getVirtualMachine()).thenReturn(vm); + when(vmProfile.getId()).thenReturn(12L); + when(vmDao.findById(12L)).thenReturn(userVm); + when(userVm.getAccountId()).thenReturn(accountId); + + when(vm.getDataCenterId()).thenReturn(dataCenterId); + when(dcDao.findById(1L)).thenReturn(mockDc); + when(avoids.shouldAvoid(mockDc)).thenReturn(false); + when(plan.getDataCenterId()).thenReturn(dataCenterId); + when(plan.getClusterId()).thenReturn(null); + when(plan.getPodId()).thenReturn(null); + + // Mock offering details. + when(vmProfile.getServiceOffering()).thenReturn(offering); + when(offering.getId()).thenReturn(offeringId); + when(vmProfile.getServiceOfferingId()).thenReturn(offeringId); + when(offering.getCpu()).thenReturn(noOfCpusInOffering); + when(offering.getSpeed()).thenReturn(cpuSpeedInOffering); + when(offering.getRamSize()).thenReturn(ramInOffering); + + List clustersWithEnoughCapacity = new ArrayList(); + clustersWithEnoughCapacity.add(1L); + clustersWithEnoughCapacity.add(2L); + clustersWithEnoughCapacity.add(3L); + clustersWithEnoughCapacity.add(4L); + clustersWithEnoughCapacity.add(5L); + clustersWithEnoughCapacity.add(6L); + + when( + capacityDao.listClustersInZoneOrPodByHostCapacities(dataCenterId, noOfCpusInOffering * cpuSpeedInOffering, ramInOffering * 1024L * 1024L, + Capacity.CAPACITY_TYPE_CPU, true)).thenReturn(clustersWithEnoughCapacity); + + Map clusterCapacityMap = new HashMap(); + clusterCapacityMap.put(1L, 2048D); + clusterCapacityMap.put(2L, 2048D); + clusterCapacityMap.put(3L, 2048D); + clusterCapacityMap.put(4L, 2048D); + clusterCapacityMap.put(5L, 2048D); + clusterCapacityMap.put(6L, 2048D); + + Pair, Map> clustersOrderedByCapacity = new Pair, Map>(clustersWithEnoughCapacity, clusterCapacityMap); + when(capacityDao.orderClustersByAggregateCapacity(dataCenterId, Capacity.CAPACITY_TYPE_CPU, true)).thenReturn(clustersOrderedByCapacity); + + List disabledClusters = new ArrayList(); + List clustersWithDisabledPods = new ArrayList(); + when(clusterDao.listDisabledClusters(dataCenterId, null)).thenReturn(disabledClusters); + when(clusterDao.listClustersWithDisabledPods(dataCenterId)).thenReturn(clustersWithDisabledPods); + + List hostList0 = new ArrayList(); + List hostList1 = new ArrayList(); + List hostList2 = new ArrayList(); + List hostList3 = new ArrayList(); + List hostList4 = new ArrayList(); + List hostList5 = new ArrayList(); + List hostList6 = new ArrayList(); + hostList0.add(new Long(1)); + hostList1.add(new Long(10)); + hostList2.add(new Long(11)); + hostList3.add(new Long(12)); + hostList4.add(new Long(13)); + hostList5.add(new Long(14)); + hostList6.add(new Long(15)); + String[] implicitHostTags = {"GPU"}; + int ramInBytes = ramInOffering * 1024 * 1024; + when(serviceOfferingDetailsDao.findDetail(Matchers.anyLong(), anyString())).thenReturn(null); + when(hostGpuGroupsDao.listHostIds()).thenReturn(hostList0); + when(capacityDao.listHostsWithEnoughCapacity(noOfCpusInOffering * cpuSpeedInOffering, ramInBytes, new Long(1), Host.Type.Routing.toString())).thenReturn(hostList1); + when(capacityDao.listHostsWithEnoughCapacity(noOfCpusInOffering * cpuSpeedInOffering, ramInBytes, new Long(2), Host.Type.Routing.toString())).thenReturn(hostList2); + when(capacityDao.listHostsWithEnoughCapacity(noOfCpusInOffering * cpuSpeedInOffering, ramInBytes, new Long(3), Host.Type.Routing.toString())).thenReturn(hostList3); + when(capacityDao.listHostsWithEnoughCapacity(noOfCpusInOffering * cpuSpeedInOffering, ramInBytes, new Long(4), Host.Type.Routing.toString())).thenReturn(hostList4); + when(capacityDao.listHostsWithEnoughCapacity(noOfCpusInOffering * cpuSpeedInOffering, ramInBytes, new Long(5), Host.Type.Routing.toString())).thenReturn(hostList5); + when(capacityDao.listHostsWithEnoughCapacity(noOfCpusInOffering * cpuSpeedInOffering, ramInBytes, new Long(6), Host.Type.Routing.toString())).thenReturn(hostList6); + when(hostTagsDao.getDistinctImplicitHostTags(hostList1, implicitHostTags)).thenReturn(Arrays.asList("abc", "pqr","xyz")); + when(hostTagsDao.getDistinctImplicitHostTags(hostList2, implicitHostTags)).thenReturn(Arrays.asList("abc", "123", "pqr", "456", "xyz")); + when(hostTagsDao.getDistinctImplicitHostTags(hostList3, implicitHostTags)).thenReturn(Arrays.asList("abc", "pqr")); + when(hostTagsDao.getDistinctImplicitHostTags(hostList4, implicitHostTags)).thenReturn(Arrays.asList("abc")); + when(hostTagsDao.getDistinctImplicitHostTags(hostList5, implicitHostTags)).thenReturn(Arrays.asList("abc", "pqr","xyz")); + when(hostTagsDao.getDistinctImplicitHostTags(hostList6, implicitHostTags)).thenReturn(Arrays.asList("abc", "123", "pqr","xyz")); + } + + @Configuration + @ComponentScan(basePackageClasses = {FirstFitPlanner.class}, + includeFilters = {@Filter(value = TestConfiguration.Library.class, type = FilterType.CUSTOM)}, + useDefaultFilters = false) + public static class TestConfiguration extends SpringUtils.CloudStackTestConfiguration { + + @Bean + public HostDao hostDao() { + return Mockito.mock(HostDao.class); + } + + @Bean + public HostTagsDao hostTagsDao() { + return Mockito.mock(HostTagsDao.class); + } + + @Bean + public HostGpuGroupsDao hostGpuGroupsDao() { + return Mockito.mock(HostGpuGroupsDao.class); + } + + @Bean + public DataCenterDao dcDao() { + return Mockito.mock(DataCenterDao.class); + } + + @Bean + public HostPodDao hostPodDao() { + return Mockito.mock(HostPodDao.class); + } + + @Bean + public ClusterDao clusterDao() { + return Mockito.mock(ClusterDao.class); + } + + @Bean + public GuestOSDao guestOsDao() { + return Mockito.mock(GuestOSDao.class); + } + + @Bean + public GuestOSCategoryDao guestOsCategoryDao() { + return Mockito.mock(GuestOSCategoryDao.class); + } + + @Bean + public DiskOfferingDao diskOfferingDao() { + return Mockito.mock(DiskOfferingDao.class); + } + + @Bean + public StoragePoolHostDao storagePoolHostDao() { + return Mockito.mock(StoragePoolHostDao.class); + } + + @Bean + public UserVmDao userVmDao() { + return Mockito.mock(UserVmDao.class); + } + + @Bean + public VMInstanceDao vmInstanceDao() { + return Mockito.mock(VMInstanceDao.class); + } + + @Bean + public VolumeDao volumeDao() { + return Mockito.mock(VolumeDao.class); + } + + @Bean + public CapacityManager capacityManager() { + return Mockito.mock(CapacityManager.class); + } + + @Bean + public ConfigurationDao configurationDao() { + return Mockito.mock(ConfigurationDao.class); + } + + @Bean + public PrimaryDataStoreDao primaryDataStoreDao() { + return Mockito.mock(PrimaryDataStoreDao.class); + } + + @Bean + public CapacityDao capacityDao() { + return Mockito.mock(CapacityDao.class); + } + + @Bean + public AccountManager accountManager() { + return Mockito.mock(AccountManager.class); + } + + @Bean + public StorageManager storageManager() { + return Mockito.mock(StorageManager.class); + } + + @Bean + public DataStoreManager dataStoreManager() { + return Mockito.mock(DataStoreManager.class); + } + + @Bean + public ClusterDetailsDao clusterDetailsDao() { + return Mockito.mock(ClusterDetailsDao.class); + } + + @Bean + public ServiceOfferingDao serviceOfferingDao() { + return Mockito.mock(ServiceOfferingDao.class); + } + + @Bean + public ServiceOfferingDetailsDao serviceOfferingDetailsDao() { + return Mockito.mock(ServiceOfferingDetailsDao.class); + } + + @Bean + public ResourceManager resourceManager() { + return Mockito.mock(ResourceManager.class); + } + + public static class Library implements TypeFilter { + @Override + public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException { + ComponentScan cs = TestConfiguration.class.getAnnotation(ComponentScan.class); + return SpringUtils.includedInBasePackageClasses(mdr.getClassMetadata().getClassName(), cs); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cloudstack/blob/e4a8bbcd/setup/db/db/schema-441to450.sql ---------------------------------------------------------------------- diff --git a/setup/db/db/schema-441to450.sql b/setup/db/db/schema-441to450.sql index 3c00948..3deb8ad 100644 --- a/setup/db/db/schema-441to450.sql +++ b/setup/db/db/schema-441to450.sql @@ -443,6 +443,8 @@ CREATE VIEW `cloud`.`template_view` AS UPDATE configuration SET value='KVM,XenServer,VMware,BareMetal,Ovm,LXC,Hyperv' WHERE name='hypervisor.list'; UPDATE `cloud`.`configuration` SET description="If set to true, will set guest VM's name as it appears on the hypervisor, to its hostname. The flag is supported for VMware hypervisor only" WHERE name='vm.instancename.flag'; +INSERT IGNORE INTO `cloud`.`configuration`(category, instance, component, name, value, description, default_value) VALUES ('Advanced', 'DEFAULT', 'management-server', 'implicit.host.tags', 'GPU', 'Tag hosts at the time of host disovery based on the host properties/capabilities ', 'GPU'); + DROP VIEW IF EXISTS `cloud`.`domain_router_view`; CREATE VIEW `cloud`.`domain_router_view` AS select