Return-Path: X-Original-To: apmail-cloudstack-commits-archive@www.apache.org Delivered-To: apmail-cloudstack-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7F196F70E for ; Thu, 9 May 2013 19:33:32 +0000 (UTC) Received: (qmail 39792 invoked by uid 500); 9 May 2013 19:33:30 -0000 Delivered-To: apmail-cloudstack-commits-archive@cloudstack.apache.org Received: (qmail 39718 invoked by uid 500); 9 May 2013 19:33:30 -0000 Mailing-List: contact commits-help@cloudstack.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cloudstack.apache.org Delivered-To: mailing list commits@cloudstack.apache.org Received: (qmail 39446 invoked by uid 99); 9 May 2013 19:33:30 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 09 May 2013 19:33:30 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 1585C88AB02; Thu, 9 May 2013 19:33:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: alena1108@apache.org To: commits@cloudstack.apache.org Date: Thu, 09 May 2013 19:33:34 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [6/9] CLOUDSTACK-747: Internal LB between VPC tiers support http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2660a6b7/plugins/network-elements/internal-loadbalancer/pom.xml ---------------------------------------------------------------------- diff --git a/plugins/network-elements/internal-loadbalancer/pom.xml b/plugins/network-elements/internal-loadbalancer/pom.xml new file mode 100644 index 0000000..48e664e --- /dev/null +++ b/plugins/network-elements/internal-loadbalancer/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + cloud-plugin-network-internallb + Apache CloudStack Plugin - Network Internal Load Balancer + + org.apache.cloudstack + cloudstack-plugins + 4.2.0-SNAPSHOT + ../../pom.xml + + + install + src + test + + + resources + + **/*.xml + + + + + + test/resources + + %regex[.*[0-9]*To[0-9]*.*Test.*] + + + + + http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2660a6b7/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/element/InternalLoadBalancerElement.java ---------------------------------------------------------------------- diff --git a/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/element/InternalLoadBalancerElement.java b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/element/InternalLoadBalancerElement.java new file mode 100644 index 0000000..4b9308b --- /dev/null +++ b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/element/InternalLoadBalancerElement.java @@ -0,0 +1,548 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.network.element; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.ejb.Local; +import javax.inject.Inject; + +import org.apache.cloudstack.api.command.admin.internallb.ConfigureInternalLoadBalancerElementCmd; +import org.apache.cloudstack.api.command.admin.internallb.CreateInternalLoadBalancerElementCmd; +import org.apache.cloudstack.api.command.admin.internallb.ListInternalLoadBalancerElementsCmd; +import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao; +import org.apache.cloudstack.network.lb.InternalLoadBalancerVMManager; +import org.apache.log4j.Logger; + +import com.cloud.agent.api.to.LoadBalancerTO; +import com.cloud.configuration.ConfigurationManager; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenter.NetworkType; +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.Network; +import com.cloud.network.Network.Capability; +import com.cloud.network.Network.Provider; +import com.cloud.network.Network.Service; +import com.cloud.network.NetworkModel; +import com.cloud.network.Networks.TrafficType; +import com.cloud.network.PhysicalNetworkServiceProvider; +import com.cloud.network.PublicIpAddress; +import com.cloud.network.VirtualRouterProvider; +import com.cloud.network.VirtualRouterProvider.VirtualRouterProviderType; +import com.cloud.network.dao.NetworkServiceMapDao; +import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; +import com.cloud.network.dao.VirtualRouterProviderDao; +import com.cloud.network.element.IpDeployer; +import com.cloud.network.element.LoadBalancingServiceProvider; +import com.cloud.network.element.NetworkElement; +import com.cloud.network.element.VirtualRouterElement; +import com.cloud.network.element.VirtualRouterProviderVO; +import com.cloud.network.lb.LoadBalancingRule; +import com.cloud.network.router.VirtualRouter; +import com.cloud.network.router.VirtualRouter.Role; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.LoadBalancerContainer; +import com.cloud.network.rules.LoadBalancerContainer.Scheme; +import com.cloud.offering.NetworkOffering; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.SearchCriteria2; +import com.cloud.utils.db.SearchCriteriaService; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; +import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.NicProfile; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.DomainRouterDao; + +@Local(value = {NetworkElement.class}) +public class InternalLoadBalancerElement extends AdapterBase implements LoadBalancingServiceProvider, InternalLoadBalancerElementService, IpDeployer{ + private static final Logger s_logger = Logger.getLogger(InternalLoadBalancerElement.class); + protected static final Map> capabilities = setCapabilities(); + private static InternalLoadBalancerElement internalLbElement = null; + + @Inject NetworkModel _ntwkModel; + @Inject NetworkServiceMapDao _ntwkSrvcDao; + @Inject DomainRouterDao _routerDao; + @Inject VirtualRouterProviderDao _vrProviderDao; + @Inject PhysicalNetworkServiceProviderDao _pNtwkSvcProviderDao; + @Inject InternalLoadBalancerVMManager _internalLbMgr; + @Inject ConfigurationManager _configMgr; + @Inject AccountManager _accountMgr; + @Inject ApplicationLoadBalancerRuleDao _appLbDao; + + protected InternalLoadBalancerElement() { + } + + + public static InternalLoadBalancerElement getInstance() { + if ( internalLbElement == null) { + internalLbElement = new InternalLoadBalancerElement(); + } + return internalLbElement; + } + + + private boolean canHandle(Network config, Scheme lbScheme) { + //works in Advance zone only + DataCenter dc = _configMgr.getZone(config.getDataCenterId()); + if (dc.getNetworkType() != NetworkType.Advanced) { + s_logger.trace("Not hanling zone of network type " + dc.getNetworkType()); + return false; + } + if (config.getGuestType() != Network.GuestType.Isolated || config.getTrafficType() != TrafficType.Guest) { + s_logger.trace("Not handling network with Type " + config.getGuestType() + " and traffic type " + config.getTrafficType()); + return false; + } + + Map lbCaps = this.getCapabilities().get(Service.Lb); + if (!lbCaps.isEmpty()) { + String schemeCaps = lbCaps.get(Capability.LbSchemes); + if (schemeCaps != null && lbScheme != null) { + if (!schemeCaps.contains(lbScheme.toString())) { + s_logger.debug("Scheme " + lbScheme.toString() + " is not supported by the provider " + this.getName()); + return false; + } + } + } + + if (!_ntwkModel.isProviderSupportServiceInNetwork(config.getId(), Service.Lb, getProvider())) { + s_logger.trace("Element " + getProvider().getName() + " doesn't support service " + Service.Lb + + " in the network " + config); + return false; + } + return true; + } + + + @Override + public Map> getCapabilities() { + return capabilities; + } + + + @Override + public Provider getProvider() { + return Provider.InternalLbVm; + } + + + @Override + public boolean implement(Network network, NetworkOffering offering, DeployDestination dest, ReservationContext context) + throws ConcurrentOperationException, ResourceUnavailableException, + InsufficientCapacityException { + + if (!canHandle(network, null)) { + s_logger.trace("No need to implement " + this.getName()); + return true; + } + + //1) Get all the Ips from the network having LB rules assigned + List ips = _appLbDao.listLbIpsBySourceIpNetworkIdAndScheme(network.getId(), Scheme.Internal); + + //2) Start those vms + for (String ip : ips) { + Ip sourceIp = new Ip(ip); + List internalLbVms; + try { + internalLbVms = _internalLbMgr.deployInternalLbVm(network, sourceIp, dest, _accountMgr.getAccount(network.getAccountId()), null); + } catch (InsufficientCapacityException e) { + s_logger.warn("Failed to deploy element " + this.getName() + " for ip " + sourceIp + " due to:", e); + return false; + } catch (ConcurrentOperationException e) { + s_logger.warn("Failed to deploy element " + this.getName() + " for ip " + sourceIp + " due to:", e); + return false; + } + + if (internalLbVms == null || internalLbVms.isEmpty()) { + throw new ResourceUnavailableException("Can't deploy " + this.getName() + " to handle LB rules", + DataCenter.class, network.getDataCenterId()); + } + } + + return true; + } + + + @Override + public boolean prepare(Network network, NicProfile nic, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) throws ConcurrentOperationException, + ResourceUnavailableException, InsufficientCapacityException { + + if (!canHandle(network, null)) { + s_logger.trace("No need to prepare " + this.getName()); + return true; + } + + if (vm.getType() == VirtualMachine.Type.User) { + //1) Get all the Ips from the network having LB rules assigned + List ips = _appLbDao.listLbIpsBySourceIpNetworkIdAndScheme(network.getId(), Scheme.Internal); + + //2) Start those vms + for (String ip : ips) { + Ip sourceIp = new Ip(ip); + List internalLbVms; + try { + internalLbVms = _internalLbMgr.deployInternalLbVm(network, sourceIp, dest, _accountMgr.getAccount(network.getAccountId()), null); + } catch (InsufficientCapacityException e) { + s_logger.warn("Failed to deploy element " + this.getName() + " for ip " + sourceIp + " due to:", e); + return false; + } catch (ConcurrentOperationException e) { + s_logger.warn("Failed to deploy element " + this.getName() + " for ip " + sourceIp + " due to:", e); + return false; + } + + if (internalLbVms == null || internalLbVms.isEmpty()) { + throw new ResourceUnavailableException("Can't deploy " + this.getName() + " to handle LB rules", + DataCenter.class, network.getDataCenterId()); + } + } + } + + return true; + } + + @Override + public boolean release(Network network, NicProfile nic, VirtualMachineProfile vm, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException { + return true; + } + + + @Override + public boolean shutdown(Network network, ReservationContext context, boolean cleanup) throws ConcurrentOperationException, ResourceUnavailableException { + List internalLbVms = _routerDao.listByNetworkAndRole(network.getId(), Role.INTERNAL_LB_VM); + if (internalLbVms == null || internalLbVms.isEmpty()) { + return true; + } + boolean result = true; + for (VirtualRouter internalLbVm : internalLbVms) { + result = result && _internalLbMgr.destroyInternalLbVm(internalLbVm.getId(), + context.getAccount(), context.getCaller().getId()); + if (cleanup) { + if (!result) { + s_logger.warn("Failed to stop internal lb element " + internalLbVm + ", but would try to process clean up anyway."); + } + result = (_internalLbMgr.destroyInternalLbVm(internalLbVm.getId(), + context.getAccount(), context.getCaller().getId())); + if (!result) { + s_logger.warn("Failed to clean up internal lb element " + internalLbVm); + } + } + } + return result; + } + + + @Override + public boolean destroy(Network network, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException { + List internalLbVms = _routerDao.listByNetworkAndRole(network.getId(), Role.INTERNAL_LB_VM); + if (internalLbVms == null || internalLbVms.isEmpty()) { + return true; + } + boolean result = true; + for (VirtualRouter internalLbVm : internalLbVms) { + result = result && (_internalLbMgr.destroyInternalLbVm(internalLbVm.getId(), + context.getAccount(), context.getCaller().getId())); + } + return result; + } + + + @Override + public boolean isReady(PhysicalNetworkServiceProvider provider) { + VirtualRouterProviderVO element = _vrProviderDao.findByNspIdAndType(provider.getId(), + VirtualRouterProviderType.InternalLbVm); + if (element == null) { + return false; + } + return element.isEnabled(); + } + + + @Override + public boolean shutdownProviderInstances(PhysicalNetworkServiceProvider provider, ReservationContext context) + throws ConcurrentOperationException, ResourceUnavailableException { + VirtualRouterProviderVO element = _vrProviderDao.findByNspIdAndType(provider.getId(), + VirtualRouterProviderType.InternalLbVm); + if (element == null) { + return true; + } + long elementId = element.getId(); + List internalLbVms = _routerDao.listByElementId(elementId); + boolean result = true; + for (DomainRouterVO internalLbVm : internalLbVms) { + result = result && (_internalLbMgr.destroyInternalLbVm(internalLbVm.getId(), + context.getAccount(), context.getCaller().getId())); + } + _vrProviderDao.remove(elementId); + + return result; + } + + + @Override + public boolean canEnableIndividualServices() { + return true; + } + + + @Override + public boolean verifyServicesCombination(Set services) { + return true; + } + + + @Override + public IpDeployer getIpDeployer(Network network) { + return this; + } + + + @Override + public boolean applyLBRules(Network network, List rules) throws ResourceUnavailableException { + //1) Get Internal LB VMs to destroy + Set vmsToDestroy = getVmsToDestroy(rules); + + //2) Get rules to apply + Map> rulesToApply = getLbRulesToApply(rules); + s_logger.debug("Applying " + rulesToApply.size() + " on element " + this.getName()); + + + for (Ip sourceIp : rulesToApply.keySet()) { + if (vmsToDestroy.contains(sourceIp)) { + //2.1 Destroy internal lb vm + List vms = _internalLbMgr.findInternalLbVms(network.getId(), sourceIp); + if (vms.size() > 0) { + //only one internal lb per IP exists + try { + s_logger.debug("Destroying internal lb vm for ip " + sourceIp.addr() + " as all the rules for this vm are in Revoke state"); + return _internalLbMgr.destroyInternalLbVm(vms.get(0).getId(), _accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM), + _accountMgr.getUserIncludingRemoved(User.UID_SYSTEM).getId()); + } catch (ConcurrentOperationException e) { + s_logger.warn("Failed to apply lb rule(s) for ip " + sourceIp.addr() + " on the element " + this.getName() + " due to:", e); + return false; + } + } + } else { + //2.2 Start Internal LB vm per IP address + List internalLbVms; + try { + DeployDestination dest = new DeployDestination(_configMgr.getZone(network.getDataCenterId()), null, null, null); + internalLbVms = _internalLbMgr.deployInternalLbVm(network, sourceIp, dest, _accountMgr.getAccount(network.getAccountId()), null); + } catch (InsufficientCapacityException e) { + s_logger.warn("Failed to apply lb rule(s) for ip " + sourceIp.addr() + "on the element " + this.getName() + " due to:", e); + return false; + } catch (ConcurrentOperationException e) { + s_logger.warn("Failed to apply lb rule(s) for ip " + sourceIp.addr() + "on the element " + this.getName() + " due to:", e); + return false; + } + + if (internalLbVms == null || internalLbVms.isEmpty()) { + throw new ResourceUnavailableException("Can't find/deploy internal lb vm to handle LB rules", + DataCenter.class, network.getDataCenterId()); + } + + //2.3 Apply Internal LB rules on the VM + if (!_internalLbMgr.applyLoadBalancingRules(network, rulesToApply.get(sourceIp), internalLbVms)) { + throw new CloudRuntimeException("Failed to apply load balancing rules for ip " + sourceIp.addr() + + " in network " + network.getId() + " on element " + this.getName()); + } + } + } + + return true; + } + + + protected Map> getLbRulesToApply(List rules) { + //Group rules by the source ip address as NetworkManager always passes the entire network lb config to the element + Map> rulesToApply = groupBySourceIp(rules); + + return rulesToApply; + } + + + protected Set getVmsToDestroy(List rules) { + //1) Group rules by the source ip address as NetworkManager always passes the entire network lb config to the element + Map> groupedRules = groupBySourceIp(rules); + + //2) Count rules in revoke state + Set vmsToDestroy = new HashSet(); + + for (Ip sourceIp : groupedRules.keySet()) { + List rulesToCheck = groupedRules.get(sourceIp); + int revoke = 0; + for (LoadBalancingRule ruleToCheck : rulesToCheck) { + if (ruleToCheck.getState() == FirewallRule.State.Revoke){ + revoke++; + } + } + + if (revoke == rulesToCheck.size()) { + s_logger.debug("Have to destroy internal lb vm for source ip " + sourceIp); + vmsToDestroy.add(sourceIp); + } + } + return vmsToDestroy; + } + + + protected Map> groupBySourceIp(List rules) { + Map> groupedRules = new HashMap>(); + for (LoadBalancingRule rule : rules) { + Ip sourceIp = rule.getSourceIp(); + if (!groupedRules.containsKey(sourceIp)) { + groupedRules.put(sourceIp, null); + } + + List rulesToApply = groupedRules.get(sourceIp); + if (rulesToApply == null) { + rulesToApply = new ArrayList(); + } + rulesToApply.add(rule); + groupedRules.put(sourceIp, rulesToApply); + } + return groupedRules; + } + + @Override + public boolean validateLBRule(Network network, LoadBalancingRule rule) { + List rules = new ArrayList(); + rules.add(rule); + if (canHandle(network, rule.getScheme())) { + List routers = _routerDao.listByNetworkAndRole(network.getId(), Role.INTERNAL_LB_VM); + if (routers == null || routers.isEmpty()) { + return true; + } + return VirtualRouterElement.validateHAProxyLBRule(rule); + } + return true; + } + + @Override + public List updateHealthChecks(Network network, List lbrules) { + return null; + } + + private static Map> setCapabilities() { + Map> capabilities = new HashMap>(); + + // Set capabilities for LB service + Map lbCapabilities = new HashMap(); + lbCapabilities.put(Capability.SupportedLBAlgorithms, "roundrobin,leastconn,source"); + lbCapabilities.put(Capability.SupportedLBIsolation, "dedicated"); + lbCapabilities.put(Capability.SupportedProtocols, "tcp, udp"); + lbCapabilities.put(Capability.SupportedStickinessMethods, VirtualRouterElement.getHAProxyStickinessCapability()); + lbCapabilities.put(Capability.LbSchemes, LoadBalancerContainer.Scheme.Internal.toString()); + + capabilities.put(Service.Lb, lbCapabilities); + return capabilities; + } + + @Override + public List> getCommands() { + List> cmdList = new ArrayList>(); + cmdList.add(CreateInternalLoadBalancerElementCmd.class); + cmdList.add(ConfigureInternalLoadBalancerElementCmd.class); + cmdList.add(ListInternalLoadBalancerElementsCmd.class); + return cmdList; + } + + @Override + public VirtualRouterProvider configureInternalLoadBalancerElement(long id, boolean enable) { + VirtualRouterProviderVO element = _vrProviderDao.findById(id); + if (element == null || element.getType() != VirtualRouterProviderType.InternalLbVm) { + throw new InvalidParameterValueException("Can't find " + this.getName() + " element with network service provider id " + id + + " to be used as a provider for " + this.getName()); + } + + element.setEnabled(enable); + element = _vrProviderDao.persist(element); + + return element; + } + + @Override + public VirtualRouterProvider addInternalLoadBalancerElement(long ntwkSvcProviderId) { + VirtualRouterProviderVO element = _vrProviderDao.findByNspIdAndType(ntwkSvcProviderId, VirtualRouterProviderType.InternalLbVm); + if (element != null) { + s_logger.debug("There is already an " + this.getName() + " with service provider id " + ntwkSvcProviderId); + return null; + } + + PhysicalNetworkServiceProvider provider = _pNtwkSvcProviderDao.findById(ntwkSvcProviderId); + if (provider == null || !provider.getProviderName().equalsIgnoreCase(this.getName())) { + throw new InvalidParameterValueException("Invalid network service provider is specified"); + } + + element = new VirtualRouterProviderVO(ntwkSvcProviderId, VirtualRouterProviderType.InternalLbVm); + element = _vrProviderDao.persist(element); + return element; + } + + + @Override + public VirtualRouterProvider getInternalLoadBalancerElement(long id) { + VirtualRouterProvider provider = _vrProviderDao.findById(id); + if (provider == null || provider.getType() != VirtualRouterProviderType.InternalLbVm) { + throw new InvalidParameterValueException("Unable to find " + this.getName() + " by id"); + } + return provider; + } + + @Override + public List searchForInternalLoadBalancerElements(Long id, Long ntwkSvsProviderId, Boolean enabled) { + + SearchCriteriaService sc = SearchCriteria2.create(VirtualRouterProviderVO.class); + if (id != null) { + sc.addAnd(sc.getEntity().getId(), Op.EQ, id); + } + if (ntwkSvsProviderId != null) { + sc.addAnd(sc.getEntity().getNspId(), Op.EQ, ntwkSvsProviderId); + } + if (enabled != null) { + sc.addAnd(sc.getEntity().isEnabled(), Op.EQ, enabled); + } + + //return only Internal LB elements + sc.addAnd(sc.getEntity().getType(), Op.EQ, VirtualRouterProvider.VirtualRouterProviderType.InternalLbVm); + + return sc.list(); + } + + @Override + public boolean applyIps(Network network, List ipAddress, Set services) throws ResourceUnavailableException { + //do nothing here; this element just has to extend the ip deployer + //as the LB service implements IPDeployerRequester + return true; + } + +} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2660a6b7/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManager.java ---------------------------------------------------------------------- diff --git a/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManager.java b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManager.java new file mode 100644 index 0000000..9faca56 --- /dev/null +++ b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManager.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.network.lb; + +import java.util.List; +import java.util.Map; + +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.Network; +import com.cloud.network.lb.LoadBalancingRule; +import com.cloud.network.router.VirtualRouter; +import com.cloud.user.Account; +import com.cloud.utils.component.Manager; +import com.cloud.utils.net.Ip; +import com.cloud.vm.VirtualMachineProfile.Param; + +public interface InternalLoadBalancerVMManager extends Manager, InternalLoadBalancerVMService{ + //RAM/CPU for the system offering used by Internal LB VMs + public static final int DEFAULT_INTERNALLB_VM_RAMSIZE = 128; // 128 MB + public static final int DEFAULT_INTERNALLB_VM_CPU_MHZ = 256; // 256 MHz + + /** + * Destroys Internal LB vm instance + * @param vmId + * @param caller + * @param callerUserId + * @return + * @throws ResourceUnavailableException + * @throws ConcurrentOperationException + */ + boolean destroyInternalLbVm(long vmId, Account caller, Long callerUserId) + throws ResourceUnavailableException, ConcurrentOperationException; + + + /** + * Deploys internal lb vm + * @param guestNetwork + * @param requestedGuestIp + * @param dest + * @param owner + * @param params + * @return + * @throws InsufficientCapacityException + * @throws ConcurrentOperationException + * @throws ResourceUnavailableException + */ + List deployInternalLbVm(Network guestNetwork, Ip requestedGuestIp, DeployDestination dest, Account owner, + Map params) throws InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException; + + + + /** + * + * @param network + * @param rules + * @param internalLbVms + * @return + * @throws ResourceUnavailableException + */ + boolean applyLoadBalancingRules(Network network, List rules, List internalLbVms) + throws ResourceUnavailableException; + + + /** + * Returns existing Internal Load Balancer elements based on guestNetworkId (required) and requestedIp (optional) + * @param guestNetworkId + * @param requestedGuestIp + * @return + */ + List findInternalLbVms(long guestNetworkId, Ip requestedGuestIp); + +} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2660a6b7/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java ---------------------------------------------------------------------- diff --git a/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java new file mode 100644 index 0000000..fe32a7b --- /dev/null +++ b/plugins/network-elements/internal-loadbalancer/src/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -0,0 +1,951 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.network.lb; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import javax.ejb.Local; +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO; +import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.AgentManager.OnError; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.GetDomRVersionAnswer; +import com.cloud.agent.api.GetDomRVersionCmd; +import com.cloud.agent.api.StopAnswer; +import com.cloud.agent.api.check.CheckSshAnswer; +import com.cloud.agent.api.check.CheckSshCommand; +import com.cloud.agent.api.routing.LoadBalancerConfigCommand; +import com.cloud.agent.api.routing.NetworkElementCommand; +import com.cloud.agent.api.to.LoadBalancerTO; +import com.cloud.agent.api.to.NicTO; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.manager.Commands; +import com.cloud.configuration.Config; +import com.cloud.configuration.dao.ConfigurationDao; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.Network; +import com.cloud.network.Network.Provider; +import com.cloud.network.Network.Service; +import com.cloud.network.NetworkManager; +import com.cloud.network.NetworkModel; +import com.cloud.network.Networks.TrafficType; +import com.cloud.network.PhysicalNetworkServiceProvider; +import com.cloud.network.VirtualRouterProvider; +import com.cloud.network.VirtualRouterProvider.VirtualRouterProviderType; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; +import com.cloud.network.dao.VirtualRouterProviderDao; +import com.cloud.network.lb.LoadBalancingRule; +import com.cloud.network.lb.LoadBalancingRule.LbDestination; +import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; +import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; +import com.cloud.network.lb.LoadBalancingRulesManager; +import com.cloud.network.router.VirtualRouter; +import com.cloud.network.router.VirtualRouter.RedundantState; +import com.cloud.network.router.VirtualRouter.Role; +import com.cloud.network.rules.FirewallRule; +import com.cloud.offering.NetworkOffering; +import com.cloud.offering.ServiceOffering; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.resource.ResourceManager; +import com.cloud.server.ConfigurationServer; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.DB; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.DomainRouterVO; +import com.cloud.vm.Nic; +import com.cloud.vm.NicProfile; +import com.cloud.vm.NicVO; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachine.State; +import com.cloud.vm.VirtualMachineGuru; +import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VirtualMachineName; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfile.Param; +import com.cloud.vm.dao.DomainRouterDao; +import com.cloud.vm.dao.NicDao; + +@Component +@Local(value = { InternalLoadBalancerVMManager.class, InternalLoadBalancerVMService.class}) +public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements + InternalLoadBalancerVMManager, VirtualMachineGuru { + private static final Logger s_logger = Logger + .getLogger(InternalLoadBalancerVMManagerImpl.class); + static final private String _internalLbVmNamePrefix = "b"; + + private String _instance; + private String _mgmtHost; + private String _mgmtCidr; + private long _internalLbVmOfferingId; + + @Inject VirtualMachineManager _itMgr; + @Inject DomainRouterDao _internalLbVmDao; + @Inject ConfigurationDao _configDao; + @Inject AgentManager _agentMgr; + @Inject DataCenterDao _dcDao; + @Inject VirtualRouterProviderDao _vrProviderDao; + @Inject ApplicationLoadBalancerRuleDao _lbDao; + @Inject NetworkModel _ntwkModel; + @Inject LoadBalancingRulesManager _lbMgr; + @Inject NicDao _nicDao; + @Inject AccountManager _accountMgr; + @Inject NetworkDao _networkDao; + @Inject NetworkManager _ntwkMgr; + @Inject ServiceOfferingDao _serviceOfferingDao; + @Inject PhysicalNetworkServiceProviderDao _physicalProviderDao; + @Inject NetworkOfferingDao _networkOfferingDao; + @Inject VMTemplateDao _templateDao; + @Inject ResourceManager _resourceMgr; + @Inject ConfigurationServer _configServer; + + @Override + public DomainRouterVO findByName(String name) { + if (!VirtualMachineName.isValidSystemVmName(name, _instance, _internalLbVmNamePrefix)) { + return null; + } + + return _internalLbVmDao.findById(VirtualMachineName.getRouterId(name)); + } + + @Override + public DomainRouterVO findById(long id) { + return _internalLbVmDao.findById(id); + } + + @Override + public DomainRouterVO persist(DomainRouterVO vm) { + DomainRouterVO virtualRouter = _internalLbVmDao.persist(vm); + return virtualRouter; + } + + @Override + public boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, + DeployDestination dest, ReservationContext context) { + + //Internal LB vm starts up with 2 Nics + //Nic #1 - Guest Nic with IP address that would act as the LB entry point + //Nic #2 - Control/Management Nic + + StringBuilder buf = profile.getBootArgsBuilder(); + buf.append(" template=domP"); + buf.append(" name=").append(profile.getHostName()); + + if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { + buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); + } + + NicProfile controlNic = null; + Network guestNetwork = null; + + for (NicProfile nic : profile.getNics()) { + int deviceId = nic.getDeviceId(); + buf.append(" eth").append(deviceId).append("ip=").append(nic.getIp4Address()); + buf.append(" eth").append(deviceId).append("mask=").append(nic.getNetmask()); + + if (nic.isDefaultNic()) { + buf.append(" gateway=").append(nic.getGateway()); + buf.append(" dns1=").append(nic.getGateway()); + } + + if (nic.getTrafficType() == TrafficType.Guest) { + guestNetwork = _ntwkModel.getNetwork(nic.getNetworkId()); + } else if (nic.getTrafficType() == TrafficType.Management) { + buf.append(" localgw=").append(dest.getPod().getGateway()); + } else if (nic.getTrafficType() == TrafficType.Control) { + controlNic = nic; + // Internal LB control command is sent over management server in VMware + if (dest.getHost().getHypervisorType() == HypervisorType.VMware) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Check if we need to add management server explicit route to Internal LB. pod cidr: " + + dest.getPod().getCidrAddress() + "/" + dest.getPod().getCidrSize() + + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + _mgmtHost); + } + + if (s_logger.isInfoEnabled()) { + s_logger.info("Add management server explicit route to Internal LB."); + } + + + buf.append(" mgmtcidr=").append(_mgmtCidr); + buf.append(" localgw=").append(dest.getPod().getGateway()); + } + } + } + + if (controlNic == null) { + throw new CloudRuntimeException("Didn't start a control port"); + } + + if (guestNetwork != null) { + String domain = guestNetwork.getNetworkDomain(); + if (domain != null) { + buf.append(" domain=" + domain); + } + } + + String type = "ilbvm"; + buf.append(" type=" + type); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Boot Args for " + profile + ": " + buf.toString()); + } + + return true; + } + + @Override + public boolean finalizeDeployment(Commands cmds, VirtualMachineProfile profile, DeployDestination dest, ReservationContext context) throws ResourceUnavailableException { + DomainRouterVO internalLbVm = profile.getVirtualMachine(); + + List nics = profile.getNics(); + for (NicProfile nic : nics) { + if (nic.getTrafficType() == TrafficType.Control) { + internalLbVm.setPrivateIpAddress(nic.getIp4Address()); + internalLbVm.setPrivateMacAddress(nic.getMacAddress()); + } + } + _internalLbVmDao.update(internalLbVm.getId(), internalLbVm); + + finalizeCommandsOnStart(cmds, profile); + return true; + } + + @Override + public boolean finalizeStart(VirtualMachineProfile profile, long hostId, Commands cmds, ReservationContext context) { + DomainRouterVO internalLbVm = profile.getVirtualMachine(); + + boolean result = true; + + Answer answer = cmds.getAnswer("checkSsh"); + if (answer != null && answer instanceof CheckSshAnswer) { + CheckSshAnswer sshAnswer = (CheckSshAnswer) answer; + if (sshAnswer == null || !sshAnswer.getResult()) { + s_logger.warn("Unable to ssh to the internal LB VM: " + sshAnswer.getDetails()); + result = false; + } + } else { + result = false; + } + if (result == false) { + return result; + } + + //Get guest network info + List guestNetworks = new ArrayList(); + List internalLbVmNics = _nicDao.listByVmId(profile.getId()); + for (Nic internalLbVmNic : internalLbVmNics) { + Network network = _ntwkModel.getNetwork(internalLbVmNic.getNetworkId()); + if (network.getTrafficType() == TrafficType.Guest) { + guestNetworks.add(network); + } + } + + answer = cmds.getAnswer("getDomRVersion"); + if (answer != null && answer instanceof GetDomRVersionAnswer) { + GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer)answer; + if (answer == null || !answer.getResult()) { + s_logger.warn("Unable to get the template/scripts version of internal LB VM " + internalLbVm.getInstanceName() + + " due to: " + versionAnswer.getDetails()); + result = false; + } else { + internalLbVm.setTemplateVersion(versionAnswer.getTemplateVersion()); + internalLbVm.setScriptsVersion(versionAnswer.getScriptsVersion()); + internalLbVm = _internalLbVmDao.persist(internalLbVm, guestNetworks); + } + } else { + result = false; + } + + return result; + } + + @Override + public boolean finalizeCommandsOnStart(Commands cmds, VirtualMachineProfile profile) { + DomainRouterVO internalLbVm = profile.getVirtualMachine(); + NicProfile controlNic = getNicProfileByTrafficType(profile, TrafficType.Control); + + if (controlNic == null) { + s_logger.error("Control network doesn't exist for the internal LB vm " + internalLbVm); + return false; + } + + finalizeSshAndVersionOnStart(cmds, profile, internalLbVm, controlNic); + + // restart network if restartNetwork = false is not specified in profile parameters + boolean reprogramGuestNtwk = true; + if (profile.getParameter(Param.ReProgramGuestNetworks) != null + && (Boolean) profile.getParameter(Param.ReProgramGuestNetworks) == false) { + reprogramGuestNtwk = false; + } + + VirtualRouterProvider lbProvider = _vrProviderDao.findById(internalLbVm.getElementId()); + if (lbProvider == null) { + throw new CloudRuntimeException("Cannot find related element " + VirtualRouterProviderType.InternalLbVm + " of vm: " + internalLbVm.getHostName()); + } + + Provider provider = Network.Provider.getProvider(lbProvider.getType().toString()); + if (provider == null) { + throw new CloudRuntimeException("Cannot find related provider of provider: " + lbProvider.getType().toString()); + } + + if (reprogramGuestNtwk) { + NicProfile guestNic = getNicProfileByTrafficType(profile, TrafficType.Guest); + finalizeLbRulesForIp(cmds, internalLbVm, provider, new Ip(guestNic.getIp4Address()), guestNic.getNetworkId()); + } + + return true; + } + + @Override + public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { + } + + @Override + public void finalizeExpunge(DomainRouterVO vm) { + } + + @Override + public Long convertToId(String vmName) { + if (!VirtualMachineName.isValidSystemVmName(vmName, _instance, _internalLbVmNamePrefix)) { + return null; + } + + return VirtualMachineName.getRouterId(vmName); + } + + @Override + public boolean plugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException, + InsufficientCapacityException { + //not supported + throw new UnsupportedOperationException("Plug nic is not supported for vm of type " + vm.getType()); + } + + @Override + public boolean unplugNic(Network network, NicTO nic, VirtualMachineTO vm, ReservationContext context, DeployDestination dest) throws ConcurrentOperationException, ResourceUnavailableException { + //not supported + throw new UnsupportedOperationException("Unplug nic is not supported for vm of type " + vm.getType()); + } + + @Override + public void prepareStop(VirtualMachineProfile profile) { + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + final Map configs = _configDao.getConfiguration("AgentManager", params); + _instance = configs.get("instance.name"); + if (_instance == null) { + _instance = "DEFAULT"; + } + + _mgmtHost = configs.get("host"); + _mgmtCidr = _configDao.getValue(Config.ManagementNetwork.key()); + + String offIdStr = configs.get(Config.InternalLbVmServiceOfferingId.key()); + if (offIdStr != null && !offIdStr.isEmpty()) { + _internalLbVmOfferingId = Long.parseLong(offIdStr); + } else { + boolean useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); + ServiceOfferingVO newOff = new ServiceOfferingVO("System Offering For Internal LB VM", 1, InternalLoadBalancerVMManager.DEFAULT_INTERNALLB_VM_RAMSIZE, InternalLoadBalancerVMManager.DEFAULT_INTERNALLB_VM_CPU_MHZ, null, + null, true, null, useLocalStorage, true, null, true, VirtualMachine.Type.InternalLoadBalancerVm, true); + newOff.setUniqueName(ServiceOffering.internalLbVmDefaultOffUniqueName); + newOff = _serviceOfferingDao.persistSystemServiceOffering(newOff); + _internalLbVmOfferingId = newOff.getId(); + } + + _itMgr.registerGuru(VirtualMachine.Type.InternalLoadBalancerVm, this); + + if (s_logger.isInfoEnabled()) { + s_logger.info(getName() + " has been configured"); + } + + return true; + } + + @Override + public String getName() { + return _name; + } + + protected NicProfile getNicProfileByTrafficType(VirtualMachineProfile profile, TrafficType trafficType) { + for (NicProfile nic : profile.getNics()) { + if (nic.getTrafficType() == trafficType && nic.getIp4Address() != null) { + return nic; + } + } + return null; + } + + protected void finalizeSshAndVersionOnStart(Commands cmds, VirtualMachineProfile profile, DomainRouterVO router, NicProfile controlNic) { + cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIp4Address(), 3922)); + + // Update internal lb vm template/scripts version + final GetDomRVersionCmd command = new GetDomRVersionCmd(); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIp4Address()); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + cmds.addCommand("getDomRVersion", command); + } + + + protected void finalizeLbRulesForIp(Commands cmds, DomainRouterVO internalLbVm, Provider provider, Ip sourceIp, long guestNtwkId) { + s_logger.debug("Resending load balancing rules as a part of start for " + internalLbVm); + List lbs = _lbDao.listBySrcIpSrcNtwkId(sourceIp, guestNtwkId); + List lbRules = new ArrayList(); + if (_ntwkModel.isProviderSupportServiceInNetwork(guestNtwkId, Service.Lb, provider)) { + // Re-apply load balancing rules + for (ApplicationLoadBalancerRuleVO lb : lbs) { + List dstList = _lbMgr.getExistingDestinations(lb.getId()); + List policyList = _lbMgr.getStickinessPolicies(lb.getId()); + List hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId()); + LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList, hcPolicyList, sourceIp); + lbRules.add(loadBalancing); + } + } + + s_logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of Intenrnal LB vm" + internalLbVm + " start."); + if (!lbRules.isEmpty()) { + createApplyLoadBalancingRulesCommands(lbRules, internalLbVm, cmds, guestNtwkId); + } + } + + private void createApplyLoadBalancingRulesCommands(List rules, VirtualRouter internalLbVm, Commands cmds, long guestNetworkId) { + + LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; + int i = 0; + boolean inline = false; + for (LoadBalancingRule rule : rules) { + boolean revoked = (rule.getState().equals(FirewallRule.State.Revoke)); + String protocol = rule.getProtocol(); + String algorithm = rule.getAlgorithm(); + String uuid = rule.getUuid(); + + String srcIp = rule.getSourceIp().addr(); + int srcPort = rule.getSourcePortStart(); + List destinations = rule.getDestinations(); + List stickinessPolicies = rule.getStickinessPolicies(); + LoadBalancerTO lb = new LoadBalancerTO(uuid, srcIp, srcPort, protocol, algorithm, revoked, false, inline, destinations, stickinessPolicies); + lbs[i++] = lb; + } + + Network guestNetwork = _ntwkModel.getNetwork(guestNetworkId); + Nic guestNic = _nicDao.findByNtwkIdAndInstanceId(guestNetwork.getId(), internalLbVm.getId()); + NicProfile guestNicProfile = new NicProfile(guestNic, guestNetwork, guestNic.getBroadcastUri(), guestNic.getIsolationUri(), + _ntwkModel.getNetworkRate(guestNetwork.getId(), internalLbVm.getId()), + _ntwkModel.isSecurityGroupSupportedInNetwork(guestNetwork), + _ntwkModel.getNetworkTag(internalLbVm.getHypervisorType(), guestNetwork)); + + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lbs, guestNic.getIp4Address(), + guestNic.getIp4Address(), internalLbVm.getPrivateIpAddress(), + _itMgr.toNicTO(guestNicProfile, internalLbVm.getHypervisorType()), internalLbVm.getVpcId()); + + cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); + cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); + cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); + cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key()); + + cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getInternalLbControlIp(internalLbVm.getId())); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, guestNic.getIp4Address()); + cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, internalLbVm.getInstanceName()); + DataCenterVO dcVo = _dcDao.findById(internalLbVm.getDataCenterId()); + cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); + cmds.addCommand(cmd); + } + + + protected String getInternalLbControlIp(long internalLbVmId) { + String controlIpAddress = null; + List nics = _nicDao.listByVmId(internalLbVmId); + for (NicVO nic : nics) { + Network ntwk = _ntwkModel.getNetwork(nic.getNetworkId()); + if (ntwk.getTrafficType() == TrafficType.Control) { + controlIpAddress = nic.getIp4Address(); + } + } + + if(controlIpAddress == null) { + s_logger.warn("Unable to find Internal LB control ip in its attached NICs!. Internal LB vm: " + internalLbVmId); + DomainRouterVO internalLbVm = _internalLbVmDao.findById(internalLbVmId); + return internalLbVm.getPrivateIpAddress(); + } + + return controlIpAddress; + } + + @Override + public boolean destroyInternalLbVm(long vmId, Account caller, Long callerUserId) + throws ResourceUnavailableException, ConcurrentOperationException { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Attempting to destroy Internal LB vm " + vmId); + } + + DomainRouterVO internalLbVm = _internalLbVmDao.findById(vmId); + if (internalLbVm == null) { + return true; + } + + _accountMgr.checkAccess(caller, null, true, internalLbVm); + + return _itMgr.expunge(internalLbVm, _accountMgr.getActiveUser(callerUserId), caller); + } + + + @Override + public VirtualRouter stopInternalLbVm(long vmId, boolean forced, Account caller, long callerUserId) throws ConcurrentOperationException, + ResourceUnavailableException { + DomainRouterVO internalLbVm = _internalLbVmDao.findById(vmId); + if (internalLbVm == null || internalLbVm.getRole() != Role.INTERNAL_LB_VM) { + throw new InvalidParameterValueException("Can't find internal lb vm by id specified"); + } + + return stopInternalLbVm(internalLbVm, forced, caller, callerUserId); + } + + protected VirtualRouter stopInternalLbVm(DomainRouterVO internalLbVm, boolean forced, Account caller, long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException { + s_logger.debug("Stopping internal lb vm " + internalLbVm); + try { + if (_itMgr.advanceStop((DomainRouterVO) internalLbVm, forced, _accountMgr.getActiveUser(callerUserId), caller)) { + return _internalLbVmDao.findById(internalLbVm.getId()); + } else { + return null; + } + } catch (OperationTimedoutException e) { + throw new CloudRuntimeException("Unable to stop " + internalLbVm, e); + } + } + + + @Override + public List deployInternalLbVm(Network guestNetwork, Ip requestedGuestIp, DeployDestination dest, + Account owner, Map params) throws InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + + List internalLbVms = findOrDeployInternalLbVm(guestNetwork, requestedGuestIp, dest, owner, params); + + return startInternalLbVms(params, internalLbVms); + } + + protected List startInternalLbVms(Map params, List internalLbVms) + throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { + List runningInternalLbVms = null; + + if (internalLbVms != null) { + runningInternalLbVms = new ArrayList(); + } else { + s_logger.debug("Have no internal lb vms to start"); + return null; + } + + for (DomainRouterVO internalLbVm : internalLbVms) { + if (internalLbVm.getState() != VirtualMachine.State.Running) { + internalLbVm = startInternalLbVm(internalLbVm, _accountMgr.getSystemAccount(), User.UID_SYSTEM, params); + } + + if (internalLbVm != null) { + runningInternalLbVms.add(internalLbVm); + } + } + return runningInternalLbVms; + } + + + + @DB + protected List findOrDeployInternalLbVm(Network guestNetwork, Ip requestedGuestIp, DeployDestination dest, + Account owner, Map params) throws ConcurrentOperationException, + InsufficientCapacityException, ResourceUnavailableException { + + List internalLbVms = new ArrayList(); + Network lock = _networkDao.acquireInLockTable(guestNetwork.getId(), _ntwkMgr.getNetworkLockTimeout()); + if (lock == null) { + throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Lock is acquired for network id " + lock.getId() + " as a part of internal lb startup in " + dest); + } + + long internalLbProviderId = getInternalLbProviderId(guestNetwork); + + try { + assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || + guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " + + guestNetwork; + assert guestNetwork.getTrafficType() == TrafficType.Guest; + + //deploy internal lb vm + Pair> planAndInternalLbVms = getDeploymentPlanAndInternalLbVms(dest, guestNetwork.getId(), requestedGuestIp); + internalLbVms = planAndInternalLbVms.second(); + DeploymentPlan plan = planAndInternalLbVms.first(); + + if (internalLbVms.size() > 0) { + s_logger.debug("Found " + internalLbVms.size() + " internal lb vms for the requested IP " + requestedGuestIp.addr()); + return internalLbVms; + } + + List> networks = createInternalLbVmNetworks(guestNetwork, plan, requestedGuestIp); + //Pass startVm=false as we are holding the network lock that needs to be released at the end of vm allocation + DomainRouterVO internalLbVm = deployInternalLbVm(owner, dest, plan, params, internalLbProviderId, _internalLbVmOfferingId, guestNetwork.getVpcId(), + networks, false); + if (internalLbVm != null) { + _internalLbVmDao.addRouterToGuestNetwork(internalLbVm, guestNetwork); + internalLbVms.add(internalLbVm); + } + } finally { + if (lock != null) { + _networkDao.releaseFromLockTable(lock.getId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Lock is released for network id " + lock.getId() + " as a part of internal lb vm startup in " + dest); + } + } + } + return internalLbVms; + } + + protected long getInternalLbProviderId(Network guestNetwork) { + VirtualRouterProviderType type = VirtualRouterProviderType.InternalLbVm; + long physicalNetworkId = _ntwkModel.getPhysicalNetworkId(guestNetwork); + + PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); + if (provider == null) { + throw new CloudRuntimeException("Cannot find service provider " + type.toString() + " in physical network " + physicalNetworkId); + } + + VirtualRouterProvider internalLbProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), type); + if (internalLbProvider == null) { + throw new CloudRuntimeException("Cannot find provider " + type.toString() + " as service provider " + provider.getId()); + } + + return internalLbProvider.getId(); + } + + protected List> createInternalLbVmNetworks(Network guestNetwork, DeploymentPlan plan, Ip guestIp) throws ConcurrentOperationException, + InsufficientAddressCapacityException { + + //Form networks + List> networks = new ArrayList>(3); + + //1) Guest network - default + if (guestNetwork != null) { + s_logger.debug("Adding nic for Internal LB in Guest network " + guestNetwork); + NicProfile guestNic = new NicProfile(); + if (guestIp != null) { + guestNic.setIp4Address(guestIp.addr()); + } else { + guestNic.setIp4Address(_ntwkMgr.acquireGuestIpAddress(guestNetwork, null)); + } + guestNic.setGateway(guestNetwork.getGateway()); + guestNic.setBroadcastUri(guestNetwork.getBroadcastUri()); + guestNic.setBroadcastType(guestNetwork.getBroadcastDomainType()); + guestNic.setIsolationUri(guestNetwork.getBroadcastUri()); + guestNic.setMode(guestNetwork.getMode()); + String gatewayCidr = guestNetwork.getCidr(); + guestNic.setNetmask(NetUtils.getCidrNetmask(gatewayCidr)); + guestNic.setDefaultNic(true); + networks.add(new Pair((NetworkVO) guestNetwork, guestNic)); + } + + //2) Control network + s_logger.debug("Adding nic for Internal LB vm in Control network "); + List offerings = _ntwkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork); + NetworkOffering controlOffering = offerings.get(0); + NetworkVO controlConfig = _ntwkMgr.setupNetwork(_accountMgr.getSystemAccount(), controlOffering, plan, null, null, false).get(0); + networks.add(new Pair(controlConfig, null)); + + return networks; + } + + + protected Pair> getDeploymentPlanAndInternalLbVms(DeployDestination dest, long guestNetworkId, Ip requestedGuestIp) { + long dcId = dest.getDataCenter().getId(); + DeploymentPlan plan = new DataCenterDeployment(dcId); + List internalLbVms = findInternalLbVms(guestNetworkId, requestedGuestIp); + + return new Pair>(plan, internalLbVms); + + } + + @Override + public List findInternalLbVms(long guestNetworkId, Ip requestedGuestIp) { + List internalLbVms = _internalLbVmDao.listByNetworkAndRole(guestNetworkId, Role.INTERNAL_LB_VM); + if (requestedGuestIp != null && !internalLbVms.isEmpty()) { + Iterator it = internalLbVms.iterator(); + while (it.hasNext()) { + DomainRouterVO vm = it.next(); + Nic nic = _nicDao.findByNtwkIdAndInstanceId(guestNetworkId, vm.getId()); + if (!nic.getIp4Address().equalsIgnoreCase(requestedGuestIp.addr())) { + it.remove(); + } + } + } + return internalLbVms; + } + + + protected DomainRouterVO deployInternalLbVm(Account owner, DeployDestination dest, DeploymentPlan plan, Map params, + long internalLbProviderId, long svcOffId, Long vpcId, + List> networks, boolean startVm) throws ConcurrentOperationException, + InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, + StorageUnavailableException, ResourceUnavailableException { + + long id = _internalLbVmDao.getNextInSequence(Long.class, "id"); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Creating the internal lb vm " + id + " in datacenter " + dest.getDataCenter()); + } + + ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(svcOffId); + + // Internal lb is the network element, we don't know the hypervisor type yet. + // Try to allocate the internal lb twice using diff hypervisors, and when failed both times, throw the exception up + List hypervisors = getHypervisors(dest, plan, null); + + int allocateRetry = 0; + int startRetry = 0; + DomainRouterVO internalLbVm = null; + for (Iterator iter = hypervisors.iterator(); iter.hasNext();) { + HypervisorType hType = iter.next(); + try { + s_logger.debug("Allocating the Internal lb with the hypervisor type " + hType); + String templateName = null; + switch (hType) { + case XenServer: + templateName = _configServer.getConfigValue(Config.RouterTemplateXen.key(), Config.ConfigurationParameterScope.zone.toString(), dest.getDataCenter().getId()); + break; + case KVM: + templateName = _configServer.getConfigValue(Config.RouterTemplateKVM.key(), Config.ConfigurationParameterScope.zone.toString(), dest.getDataCenter().getId()); + break; + case VMware: + templateName = _configServer.getConfigValue(Config.RouterTemplateVmware.key(), Config.ConfigurationParameterScope.zone.toString(), dest.getDataCenter().getId()); + break; + case Hyperv: + templateName = _configServer.getConfigValue(Config.RouterTemplateHyperv.key(), Config.ConfigurationParameterScope.zone.toString(), dest.getDataCenter().getId()); + break; + case LXC: + templateName = _configServer.getConfigValue(Config.RouterTemplateLXC.key(), Config.ConfigurationParameterScope.zone.toString(), dest.getDataCenter().getId()); + break; + default: break; + } + VMTemplateVO template = _templateDao.findRoutingTemplate(hType, templateName); + + if (template == null) { + s_logger.debug(hType + " won't support system vm, skip it"); + continue; + } + + internalLbVm = new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, + VirtualMachineName.getSystemVmName(id, _instance, _internalLbVmNamePrefix), template.getId(), template.getHypervisorType(), + template.getGuestOSId(), owner.getDomainId(), owner.getId(), false, 0, false, + RedundantState.UNKNOWN, false, false, VirtualMachine.Type.InternalLoadBalancerVm, vpcId); + internalLbVm.setRole(Role.INTERNAL_LB_VM); + internalLbVm = _itMgr.allocate(internalLbVm, template, routerOffering, networks, plan, null, owner); + } catch (InsufficientCapacityException ex) { + if (allocateRetry < 2 && iter.hasNext()) { + s_logger.debug("Failed to allocate the Internal lb vm with hypervisor type " + hType + ", retrying one more time"); + continue; + } else { + throw ex; + } + } finally { + allocateRetry++; + } + + if (startVm) { + try { + internalLbVm = startInternalLbVm(internalLbVm, _accountMgr.getSystemAccount(), User.UID_SYSTEM, params); + break; + } catch (InsufficientCapacityException ex) { + if (startRetry < 2 && iter.hasNext()) { + s_logger.debug("Failed to start the Internal lb vm " + internalLbVm + " with hypervisor type " + hType + ", " + + "destroying it and recreating one more time"); + // destroy the internal lb vm + destroyInternalLbVm(internalLbVm.getId(), _accountMgr.getSystemAccount(), User.UID_SYSTEM); + continue; + } else { + throw ex; + } + } finally { + startRetry++; + } + } else { + //return stopped internal lb vm + return internalLbVm; + } + } + return internalLbVm; + } + + + + protected DomainRouterVO startInternalLbVm(DomainRouterVO internalLbVm, Account caller, long callerUserId, Map params) + throws StorageUnavailableException, InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + s_logger.debug("Starting Internal LB VM " + internalLbVm); + if (_itMgr.start(internalLbVm, params, _accountMgr.getUserIncludingRemoved(callerUserId), caller, null) != null) { + if (internalLbVm.isStopPending()) { + s_logger.info("Clear the stop pending flag of Internal LB VM " + internalLbVm.getHostName() + " after start router successfully!"); + internalLbVm.setStopPending(false); + internalLbVm = _internalLbVmDao.persist(internalLbVm); + } + return _internalLbVmDao.findById(internalLbVm.getId()); + } else { + return null; + } + } + + + protected List getHypervisors(DeployDestination dest, DeploymentPlan plan, + List supportedHypervisors) throws InsufficientServerCapacityException { + List hypervisors = new ArrayList(); + + HypervisorType defaults = _resourceMgr.getDefaultHypervisor(dest.getDataCenter().getId()); + if (defaults != HypervisorType.None) { + hypervisors.add(defaults); + } else { + //if there is no default hypervisor, get it from the cluster + hypervisors = _resourceMgr.getSupportedHypervisorTypes(dest.getDataCenter().getId(), true, + plan.getPodId()); + } + + //keep only elements defined in supported hypervisors + StringBuilder hTypesStr = new StringBuilder(); + if (supportedHypervisors != null && !supportedHypervisors.isEmpty()) { + hypervisors.retainAll(supportedHypervisors); + for (HypervisorType hType : supportedHypervisors) { + hTypesStr.append(hType).append(" "); + } + } + + if (hypervisors.isEmpty()) { + throw new InsufficientServerCapacityException("Unable to create internal lb vm, " + + "there are no clusters in the zone ", DataCenter.class, dest.getDataCenter().getId()); + } + return hypervisors; + } + + @Override + public boolean applyLoadBalancingRules(Network network, final List rules, List internalLbVms) + throws ResourceUnavailableException { + if (rules == null || rules.isEmpty()) { + s_logger.debug("No lb rules to be applied for network " + network); + return true; + } + + //only one internal lb vm is supported per ip address at this time + if (internalLbVms == null || internalLbVms.isEmpty()) { + throw new CloudRuntimeException("Can't apply the lb rules on network " + network + " as the list of internal lb vms is empty"); + } + + VirtualRouter lbVm = internalLbVms.get(0); + if (lbVm.getState() == State.Running) { + return sendLBRules(lbVm, rules, network.getId()); + } else if (lbVm.getState() == State.Stopped || lbVm.getState() == State.Stopping) { + s_logger.debug("Internal LB VM " + lbVm.getInstanceName() + " is in " + lbVm.getState() + + ", so not sending apply lb rules commands to the backend"); + return true; + } else { + s_logger.warn("Unable to apply lb rules, Internal LB VM is not in the right state " + lbVm.getState()); + throw new ResourceUnavailableException("Unable to apply lb rules; Internal LB VM is not in the right state", DataCenter.class, lbVm.getDataCenterId()); + } + } + + protected boolean sendLBRules(VirtualRouter internalLbVm, List rules, long guestNetworkId) throws ResourceUnavailableException { + Commands cmds = new Commands(OnError.Continue); + createApplyLoadBalancingRulesCommands(rules, internalLbVm, cmds, guestNetworkId); + return sendCommandsToInternalLbVm(internalLbVm, cmds); + } + + + protected boolean sendCommandsToInternalLbVm(final VirtualRouter internalLbVm, Commands cmds) throws AgentUnavailableException { + Answer[] answers = null; + try { + answers = _agentMgr.send(internalLbVm.getHostId(), cmds); + } catch (OperationTimedoutException e) { + s_logger.warn("Timed Out", e); + throw new AgentUnavailableException("Unable to send commands to virtual router ", internalLbVm.getHostId(), e); + } + + if (answers == null) { + return false; + } + + if (answers.length != cmds.size()) { + return false; + } + + boolean result = true; + if (answers.length > 0) { + for (Answer answer : answers) { + if (!answer.getResult()) { + result = false; + break; + } + } + } + return result; + } + + + @Override + public VirtualRouter startInternalLbVm(long internalLbVmId, Account caller, long callerUserId) + throws StorageUnavailableException, InsufficientCapacityException, + ConcurrentOperationException, ResourceUnavailableException { + + DomainRouterVO internalLbVm = _internalLbVmDao.findById(internalLbVmId); + if (internalLbVm == null || internalLbVm.getRole() != Role.INTERNAL_LB_VM) { + throw new InvalidParameterValueException("Can't find internal lb vm by id specified"); + } + + return startInternalLbVm(internalLbVm, caller, callerUserId, null); + } +} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/2660a6b7/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbelement/ElementChildTestConfiguration.java ---------------------------------------------------------------------- diff --git a/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbelement/ElementChildTestConfiguration.java b/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbelement/ElementChildTestConfiguration.java new file mode 100644 index 0000000..6959b95 --- /dev/null +++ b/plugins/network-elements/internal-loadbalancer/test/org/apache/cloudstack/internallbelement/ElementChildTestConfiguration.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.internallbelement; + +import java.io.IOException; + +import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao; +import org.apache.cloudstack.network.lb.InternalLoadBalancerVMManager; +import org.apache.cloudstack.test.utils.SpringUtils; +import org.mockito.Mockito; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.ComponentScan.Filter; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.core.type.filter.TypeFilter; + +import com.cloud.configuration.ConfigurationManager; +import com.cloud.dc.dao.AccountVlanMapDaoImpl; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.network.NetworkManager; +import com.cloud.network.NetworkModel; +import com.cloud.network.dao.NetworkServiceMapDao; +import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; +import com.cloud.network.dao.VirtualRouterProviderDao; +import com.cloud.user.AccountManager; +import com.cloud.vm.dao.DomainRouterDao; + +@Configuration +@ComponentScan( + basePackageClasses={ + AccountVlanMapDaoImpl.class + }, + includeFilters={@Filter(value=ElementChildTestConfiguration.Library.class, type=FilterType.CUSTOM)}, + useDefaultFilters=false + ) + +public class ElementChildTestConfiguration { + public static class Library implements TypeFilter { + @Bean + public AccountManager accountManager() { + return Mockito.mock(AccountManager.class); + } + + + @Bean + public DomainRouterDao domainRouterDao() { + return Mockito.mock(DomainRouterDao.class); + } + + @Bean + public VirtualRouterProviderDao virtualRouterProviderDao() { + return Mockito.mock(VirtualRouterProviderDao.class); + } + + @Bean + public NetworkModel networkModel() { + return Mockito.mock(NetworkModel.class); + } + + + @Bean + public NetworkManager networkManager() { + return Mockito.mock(NetworkManager.class); + } + + + @Bean + public PhysicalNetworkServiceProviderDao physicalNetworkServiceProviderDao() { + return Mockito.mock(PhysicalNetworkServiceProviderDao.class); + } + + @Bean + public NetworkServiceMapDao networkServiceMapDao() { + return Mockito.mock(NetworkServiceMapDao.class); + } + + @Bean + public InternalLoadBalancerVMManager internalLoadBalancerVMManager() { + return Mockito.mock(InternalLoadBalancerVMManager.class); + } + + @Bean + public ConfigurationManager confugurationManager() { + return Mockito.mock(ConfigurationManager.class); + } + + + @Bean + public ApplicationLoadBalancerRuleDao applicationLoadBalancerRuleDao() { + return Mockito.mock(ApplicationLoadBalancerRuleDao.class); + } + + @Bean + public DataCenterDao dataCenterDao() { + return Mockito.mock(DataCenterDao.class); + } + + + + @Override + public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException { + mdr.getClassMetadata().getClassName(); + ComponentScan cs = ElementChildTestConfiguration.class.getAnnotation(ComponentScan.class); + return SpringUtils.includedInBasePackageClasses(mdr.getClassMetadata().getClassName(), cs); + } + } +}