| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| package org.apache.cloudstack.network.lb; |
| |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Iterator; |
| import java.util.LinkedHashMap; |
| import java.util.List; |
| import java.util.Map; |
| |
| import javax.inject.Inject; |
| import javax.naming.ConfigurationException; |
| |
| import org.apache.cloudstack.context.CallContext; |
| import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; |
| import org.apache.cloudstack.framework.config.dao.ConfigurationDao; |
| import org.apache.cloudstack.lb.ApplicationLoadBalancerRuleVO; |
| import org.apache.cloudstack.lb.dao.ApplicationLoadBalancerRuleDao; |
| |
| import com.cloud.agent.AgentManager; |
| import com.cloud.agent.api.Answer; |
| import com.cloud.agent.api.Command; |
| import com.cloud.agent.api.GetDomRVersionAnswer; |
| import com.cloud.agent.api.GetDomRVersionCmd; |
| import com.cloud.agent.api.check.CheckSshAnswer; |
| import com.cloud.agent.api.check.CheckSshCommand; |
| import com.cloud.agent.api.routing.LoadBalancerConfigCommand; |
| import com.cloud.agent.api.routing.NetworkElementCommand; |
| import com.cloud.agent.api.to.LoadBalancerTO; |
| import com.cloud.agent.manager.Commands; |
| import com.cloud.configuration.Config; |
| import com.cloud.configuration.ConfigurationManagerImpl; |
| import com.cloud.dc.DataCenter; |
| import com.cloud.dc.DataCenterVO; |
| import com.cloud.dc.dao.DataCenterDao; |
| import com.cloud.deploy.DataCenterDeployment; |
| import com.cloud.deploy.DeployDestination; |
| import com.cloud.deploy.DeploymentPlan; |
| import com.cloud.exception.AgentUnavailableException; |
| import com.cloud.exception.ConcurrentOperationException; |
| import com.cloud.exception.InsufficientAddressCapacityException; |
| import com.cloud.exception.InsufficientCapacityException; |
| import com.cloud.exception.InsufficientServerCapacityException; |
| import com.cloud.exception.InvalidParameterValueException; |
| import com.cloud.exception.OperationTimedoutException; |
| import com.cloud.exception.ResourceUnavailableException; |
| import com.cloud.exception.StorageUnavailableException; |
| import com.cloud.hypervisor.Hypervisor.HypervisorType; |
| import com.cloud.network.IpAddressManager; |
| import com.cloud.network.Network; |
| import com.cloud.network.Network.Provider; |
| import com.cloud.network.Network.Service; |
| import com.cloud.network.NetworkModel; |
| import com.cloud.network.Networks.TrafficType; |
| import com.cloud.network.PhysicalNetworkServiceProvider; |
| import com.cloud.network.VirtualRouterProvider; |
| import com.cloud.network.VirtualRouterProvider.Type; |
| import com.cloud.network.dao.NetworkDao; |
| import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; |
| import com.cloud.network.dao.VirtualRouterProviderDao; |
| import com.cloud.network.lb.LoadBalancingRule; |
| import com.cloud.network.lb.LoadBalancingRule.LbDestination; |
| import com.cloud.network.lb.LoadBalancingRule.LbHealthCheckPolicy; |
| import com.cloud.network.lb.LoadBalancingRule.LbStickinessPolicy; |
| import com.cloud.network.lb.LoadBalancingRulesManager; |
| import com.cloud.network.router.VirtualNetworkApplianceManager; |
| import com.cloud.network.router.VirtualRouter; |
| import com.cloud.network.router.VirtualRouter.RedundantState; |
| import com.cloud.network.router.VirtualRouter.Role; |
| import com.cloud.network.rules.FirewallRule; |
| import com.cloud.offering.NetworkOffering; |
| import com.cloud.offering.ServiceOffering; |
| import com.cloud.offerings.dao.NetworkOfferingDao; |
| import com.cloud.resource.ResourceManager; |
| import com.cloud.service.ServiceOfferingVO; |
| import com.cloud.service.dao.ServiceOfferingDao; |
| import com.cloud.storage.Storage; |
| import com.cloud.storage.VMTemplateVO; |
| import com.cloud.storage.dao.VMTemplateDao; |
| import com.cloud.user.Account; |
| import com.cloud.user.AccountManager; |
| import com.cloud.user.User; |
| import com.cloud.user.UserVO; |
| import com.cloud.user.dao.UserDao; |
| import com.cloud.utils.Pair; |
| import com.cloud.utils.component.ManagerBase; |
| import com.cloud.utils.db.DB; |
| import com.cloud.utils.exception.CloudRuntimeException; |
| import com.cloud.utils.net.Ip; |
| import com.cloud.utils.net.NetUtils; |
| import com.cloud.vm.DomainRouterVO; |
| import com.cloud.vm.Nic; |
| import com.cloud.vm.NicProfile; |
| import com.cloud.vm.NicVO; |
| import com.cloud.vm.ReservationContext; |
| import com.cloud.vm.VirtualMachine; |
| import com.cloud.vm.VirtualMachine.State; |
| import com.cloud.vm.VirtualMachineGuru; |
| import com.cloud.vm.VirtualMachineManager; |
| import com.cloud.vm.VirtualMachineName; |
| import com.cloud.vm.VirtualMachineProfile; |
| import com.cloud.vm.VirtualMachineProfile.Param; |
| import com.cloud.vm.dao.DomainRouterDao; |
| import com.cloud.vm.dao.NicDao; |
| |
| import static com.cloud.hypervisor.Hypervisor.HypervisorType.Hyperv; |
| import static com.cloud.hypervisor.Hypervisor.HypervisorType.KVM; |
| import static com.cloud.hypervisor.Hypervisor.HypervisorType.LXC; |
| import static com.cloud.hypervisor.Hypervisor.HypervisorType.VMware; |
| import static com.cloud.hypervisor.Hypervisor.HypervisorType.XenServer; |
| |
| public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements InternalLoadBalancerVMManager, InternalLoadBalancerVMService, VirtualMachineGuru { |
| static final private String InternalLbVmNamePrefix = "b"; |
| |
| private String _instance; |
| private String _mgmtHost; |
| private String _mgmtCidr; |
| private long _internalLbVmOfferingId = 0L; |
| |
| @Inject |
| IpAddressManager _ipAddrMgr; |
| @Inject |
| VirtualMachineManager _itMgr; |
| @Inject |
| DomainRouterDao _internalLbVmDao; |
| @Inject |
| ConfigurationDao _configDao; |
| @Inject |
| AgentManager _agentMgr; |
| @Inject |
| DataCenterDao _dcDao; |
| @Inject |
| VirtualRouterProviderDao _vrProviderDao; |
| @Inject |
| ApplicationLoadBalancerRuleDao _lbDao; |
| @Inject |
| NetworkModel _ntwkModel; |
| @Inject |
| LoadBalancingRulesManager _lbMgr; |
| @Inject |
| NicDao _nicDao; |
| @Inject |
| AccountManager _accountMgr; |
| @Inject |
| NetworkDao _networkDao; |
| @Inject |
| NetworkOrchestrationService _ntwkMgr; |
| @Inject |
| ServiceOfferingDao _serviceOfferingDao; |
| @Inject |
| PhysicalNetworkServiceProviderDao _physicalProviderDao; |
| @Inject |
| NetworkOfferingDao _networkOfferingDao; |
| @Inject |
| VMTemplateDao _templateDao; |
| @Inject |
| ResourceManager _resourceMgr; |
| @Inject |
| UserDao _userDao; |
| |
| @Override |
| public boolean finalizeVirtualMachineProfile(final VirtualMachineProfile profile, final DeployDestination dest, final ReservationContext context) { |
| |
| //Internal LB vm starts up with 2 Nics |
| //Nic #1 - Guest Nic with IP address that would act as the LB entry point |
| //Nic #2 - Control/Management Nic |
| |
| final StringBuilder buf = profile.getBootArgsBuilder(); |
| buf.append(" template=domP"); |
| buf.append(" name=").append(profile.getHostName()); |
| |
| if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { |
| buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); |
| } |
| |
| NicProfile controlNic = null; |
| Network guestNetwork = null; |
| |
| for (final NicProfile nic : profile.getNics()) { |
| final int deviceId = nic.getDeviceId(); |
| buf.append(" eth").append(deviceId).append("ip=").append(nic.getIPv4Address()); |
| buf.append(" eth").append(deviceId).append("mask=").append(nic.getIPv4Netmask()); |
| |
| if (nic.isDefaultNic()) { |
| buf.append(" gateway=").append(nic.getIPv4Gateway()); |
| buf.append(" dns1=").append(nic.getIPv4Gateway()); |
| } |
| |
| if (nic.getTrafficType() == TrafficType.Guest) { |
| guestNetwork = _ntwkModel.getNetwork(nic.getNetworkId()); |
| } else if (nic.getTrafficType() == TrafficType.Management) { |
| buf.append(" localgw=").append(dest.getPod().getGateway()); |
| } else if (nic.getTrafficType() == TrafficType.Control) { |
| controlNic = nic; |
| // Internal LB control command is sent over management server in VMware |
| if (dest.getHost().getHypervisorType() == VMware) { |
| if (logger.isInfoEnabled()) { |
| logger.info("Check if we need to add management server explicit route to Internal LB. pod cidr: " + dest.getPod().getCidrAddress() + "/" + |
| dest.getPod().getCidrSize() + ", pod gateway: " + dest.getPod().getGateway() + ", management host: " + _mgmtHost); |
| } |
| |
| if (logger.isInfoEnabled()) { |
| logger.info("Add management server explicit route to Internal LB."); |
| } |
| |
| buf.append(" mgmtcidr=").append(_mgmtCidr); |
| buf.append(" localgw=").append(dest.getPod().getGateway()); |
| } |
| } |
| String msPublicKey = _configDao.getValue("ssh.publickey"); |
| buf.append(" authorized_key=").append(VirtualMachineGuru.getEncodedMsPublicKey(msPublicKey)); |
| } |
| |
| if (controlNic == null) { |
| throw new CloudRuntimeException("Didn't start a control port"); |
| } |
| |
| if (guestNetwork != null) { |
| final String domain = guestNetwork.getNetworkDomain(); |
| if (domain != null) { |
| buf.append(" domain=" + domain); |
| } |
| } |
| |
| final String type = "ilbvm"; |
| buf.append(" type=" + type); |
| |
| if (logger.isDebugEnabled()) { |
| logger.debug("Boot Args for " + profile + ": " + buf.toString()); |
| } |
| |
| return true; |
| } |
| |
| @Override |
| public boolean finalizeDeployment(final Commands cmds, final VirtualMachineProfile profile, final DeployDestination dest, final ReservationContext context) |
| throws ResourceUnavailableException { |
| |
| final DomainRouterVO internalLbVm = _internalLbVmDao.findById(profile.getId()); |
| |
| final List<NicProfile> nics = profile.getNics(); |
| for (final NicProfile nic : nics) { |
| if (nic.getTrafficType() == TrafficType.Control) { |
| internalLbVm.setPrivateIpAddress(nic.getIPv4Address()); |
| internalLbVm.setPrivateMacAddress(nic.getMacAddress()); |
| } |
| } |
| _internalLbVmDao.update(internalLbVm.getId(), internalLbVm); |
| |
| finalizeCommandsOnStart(cmds, profile); |
| return true; |
| } |
| |
| @Override |
| public boolean finalizeStart(final VirtualMachineProfile profile, final long hostId, final Commands cmds, final ReservationContext context) { |
| DomainRouterVO internalLbVm = _internalLbVmDao.findById(profile.getId()); |
| |
| boolean result = true; |
| |
| Answer answer = cmds.getAnswer("checkSsh"); |
| if (answer != null && answer instanceof CheckSshAnswer) { |
| final CheckSshAnswer sshAnswer = (CheckSshAnswer)answer; |
| if (sshAnswer == null || !sshAnswer.getResult()) { |
| logger.warn("Unable to ssh to the internal LB VM: " + sshAnswer.getDetails()); |
| result = false; |
| } |
| } else { |
| result = false; |
| } |
| if (result == false) { |
| return result; |
| } |
| |
| //Get guest network info |
| final List<Network> guestNetworks = new ArrayList<Network>(); |
| final List<? extends Nic> internalLbVmNics = _nicDao.listByVmId(profile.getId()); |
| for (final Nic internalLbVmNic : internalLbVmNics) { |
| final Network network = _ntwkModel.getNetwork(internalLbVmNic.getNetworkId()); |
| if (network.getTrafficType() == TrafficType.Guest) { |
| guestNetworks.add(network); |
| } |
| } |
| |
| answer = cmds.getAnswer("getDomRVersion"); |
| if (answer != null && answer instanceof GetDomRVersionAnswer) { |
| final GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer)answer; |
| if (answer == null || !answer.getResult()) { |
| logger.warn("Unable to get the template/scripts version of internal LB VM " + internalLbVm.getInstanceName() + " due to: " + versionAnswer.getDetails()); |
| result = false; |
| } else { |
| internalLbVm.setTemplateVersion(versionAnswer.getTemplateVersion()); |
| internalLbVm.setScriptsVersion(versionAnswer.getScriptsVersion()); |
| internalLbVm = _internalLbVmDao.persist(internalLbVm, guestNetworks); |
| } |
| } else { |
| result = false; |
| } |
| |
| return result; |
| } |
| |
| @Override |
| public boolean finalizeCommandsOnStart(final Commands cmds, final VirtualMachineProfile profile) { |
| final DomainRouterVO internalLbVm = _internalLbVmDao.findById(profile.getId()); |
| final NicProfile controlNic = getNicProfileByTrafficType(profile, TrafficType.Control); |
| |
| if (controlNic == null) { |
| logger.error("Control network doesn't exist for the internal LB vm " + internalLbVm); |
| return false; |
| } |
| |
| finalizeSshAndVersionOnStart(cmds, profile, internalLbVm, controlNic); |
| |
| // restart network if restartNetwork = false is not specified in profile parameters |
| boolean reprogramGuestNtwk = true; |
| if (profile.getParameter(Param.ReProgramGuestNetworks) != null && (Boolean)profile.getParameter(Param.ReProgramGuestNetworks) == false) { |
| reprogramGuestNtwk = false; |
| } |
| |
| final VirtualRouterProvider lbProvider = _vrProviderDao.findById(internalLbVm.getElementId()); |
| if (lbProvider == null) { |
| throw new CloudRuntimeException("Cannot find related element " + Type.InternalLbVm + " of vm: " + internalLbVm.getHostName()); |
| } |
| |
| final Provider provider = Network.Provider.getProvider(lbProvider.getType().toString()); |
| if (provider == null) { |
| throw new CloudRuntimeException("Cannot find related provider of provider: " + lbProvider.getType().toString()); |
| } |
| |
| if (reprogramGuestNtwk) { |
| final NicProfile guestNic = getNicProfileByTrafficType(profile, TrafficType.Guest); |
| finalizeLbRulesForIp(cmds, internalLbVm, provider, new Ip(guestNic.getIPv4Address()), guestNic.getNetworkId()); |
| } |
| |
| return true; |
| } |
| |
| @Override |
| public void finalizeStop(final VirtualMachineProfile profile, final Answer answer) { |
| } |
| |
| @Override |
| public void finalizeExpunge(final VirtualMachine vm) { |
| } |
| |
| @Override |
| public void prepareStop(final VirtualMachineProfile profile) { |
| } |
| |
| @Override |
| public void finalizeUnmanage(VirtualMachine vm) { |
| } |
| |
| @Override |
| public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException { |
| final Map<String, String> configs = _configDao.getConfiguration("AgentManager", params); |
| _instance = configs.get("instance.name"); |
| if (_instance == null) { |
| _instance = "DEFAULT"; |
| } |
| |
| _mgmtHost = configs.get("host"); |
| _mgmtCidr = _configDao.getValue(Config.ManagementNetwork.key()); |
| |
| final String offUUID = configs.get(Config.InternalLbVmServiceOfferingId.key()); |
| if (offUUID != null && !offUUID.isEmpty()) { |
| //get the id by offering UUID |
| final ServiceOfferingVO off = _serviceOfferingDao.findByUuid(offUUID); |
| if (off != null) { |
| _internalLbVmOfferingId = off.getId(); |
| } else { |
| logger.warn("Invalid offering UUID is passed in " + Config.InternalLbVmServiceOfferingId.key() + "; the default offering will be used instead"); |
| } |
| } |
| |
| //if offering wasn't set, try to get the default one |
| if (_internalLbVmOfferingId == 0L) { |
| List<ServiceOfferingVO> offerings = _serviceOfferingDao.createSystemServiceOfferings("System Offering For Internal LB VM", |
| ServiceOffering.internalLbVmDefaultOffUniqueName, 1, InternalLoadBalancerVMManager.DEFAULT_INTERNALLB_VM_RAMSIZE, |
| InternalLoadBalancerVMManager.DEFAULT_INTERNALLB_VM_CPU_MHZ, null, null, true, null, |
| Storage.ProvisioningType.THIN, true, null, true, VirtualMachine.Type.InternalLoadBalancerVm, true); |
| if (offerings == null || offerings.size() < 2) { |
| String msg = "Data integrity problem : System Offering For Internal LB VM has been removed?"; |
| logger.error(msg); |
| throw new ConfigurationException(msg); |
| } |
| } |
| |
| _itMgr.registerGuru(VirtualMachine.Type.InternalLoadBalancerVm, this); |
| |
| if (logger.isInfoEnabled()) { |
| logger.info(getName() + " has been configured"); |
| } |
| |
| return true; |
| } |
| |
| @Override |
| public String getName() { |
| return _name; |
| } |
| |
| protected NicProfile getNicProfileByTrafficType(final VirtualMachineProfile profile, final TrafficType trafficType) { |
| for (final NicProfile nic : profile.getNics()) { |
| if (nic.getTrafficType() == trafficType && nic.getIPv4Address() != null) { |
| return nic; |
| } |
| } |
| return null; |
| } |
| |
| protected void finalizeSshAndVersionOnStart(final Commands cmds, final VirtualMachineProfile profile, final DomainRouterVO router, final NicProfile controlNic) { |
| cmds.addCommand("checkSsh", new CheckSshCommand(profile.getInstanceName(), controlNic.getIPv4Address(), 3922)); |
| |
| // Update internal lb vm template/scripts version |
| final GetDomRVersionCmd command = new GetDomRVersionCmd(); |
| command.setAccessDetail(NetworkElementCommand.ROUTER_IP, controlNic.getIPv4Address()); |
| command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); |
| cmds.addCommand("getDomRVersion", command); |
| } |
| |
| protected void finalizeLbRulesForIp(final Commands cmds, final DomainRouterVO internalLbVm, final Provider provider, final Ip sourceIp, final long guestNtwkId) { |
| logger.debug("Resending load balancing rules as a part of start for " + internalLbVm); |
| final List<ApplicationLoadBalancerRuleVO> lbs = _lbDao.listBySrcIpSrcNtwkId(sourceIp, guestNtwkId); |
| final List<LoadBalancingRule> lbRules = new ArrayList<LoadBalancingRule>(); |
| if (_ntwkModel.isProviderSupportServiceInNetwork(guestNtwkId, Service.Lb, provider)) { |
| // Re-apply load balancing rules |
| for (final ApplicationLoadBalancerRuleVO lb : lbs) { |
| final List<LbDestination> dstList = _lbMgr.getExistingDestinations(lb.getId()); |
| final List<LbStickinessPolicy> policyList = _lbMgr.getStickinessPolicies(lb.getId()); |
| final List<LbHealthCheckPolicy> hcPolicyList = _lbMgr.getHealthCheckPolicies(lb.getId()); |
| final LoadBalancingRule loadBalancing = new LoadBalancingRule(lb, dstList, policyList, hcPolicyList, sourceIp); |
| lbRules.add(loadBalancing); |
| } |
| } |
| |
| logger.debug("Found " + lbRules.size() + " load balancing rule(s) to apply as a part of Intenrnal LB vm" + internalLbVm + " start."); |
| if (!lbRules.isEmpty()) { |
| createApplyLoadBalancingRulesCommands(lbRules, internalLbVm, cmds, guestNtwkId); |
| } |
| } |
| |
| private void createApplyLoadBalancingRulesCommands(final List<LoadBalancingRule> rules, final VirtualRouter internalLbVm, final Commands cmds, final long guestNetworkId) { |
| |
| final LoadBalancerTO[] lbs = new LoadBalancerTO[rules.size()]; |
| int i = 0; |
| final boolean inline = false; |
| for (final LoadBalancingRule rule : rules) { |
| final boolean revoked = rule.getState().equals(FirewallRule.State.Revoke); |
| final String protocol = rule.getProtocol(); |
| final String algorithm = rule.getAlgorithm(); |
| final String uuid = rule.getUuid(); |
| |
| final String srcIp = rule.getSourceIp().addr(); |
| final int srcPort = rule.getSourcePortStart(); |
| final List<LbDestination> destinations = rule.getDestinations(); |
| final List<LbStickinessPolicy> stickinessPolicies = rule.getStickinessPolicies(); |
| final LoadBalancerTO lb = new LoadBalancerTO(uuid, srcIp, srcPort, protocol, algorithm, revoked, false, inline, destinations, stickinessPolicies); |
| lb.setCidrList(rule.getCidrList()); |
| lbs[i++] = lb; |
| } |
| |
| final Network guestNetwork = _ntwkModel.getNetwork(guestNetworkId); |
| final Nic guestNic = _nicDao.findByNtwkIdAndInstanceId(guestNetwork.getId(), internalLbVm.getId()); |
| final NicProfile guestNicProfile = |
| new NicProfile(guestNic, guestNetwork, guestNic.getBroadcastUri(), guestNic.getIsolationUri(), _ntwkModel.getNetworkRate(guestNetwork.getId(), |
| internalLbVm.getId()), _ntwkModel.isSecurityGroupSupportedInNetwork(guestNetwork), _ntwkModel.getNetworkTag(internalLbVm.getHypervisorType(), |
| guestNetwork)); |
| |
| final NetworkOffering offering = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()); |
| String maxconn = null; |
| if (offering.getConcurrentConnections() == null) { |
| maxconn = _configDao.getValue(Config.NetworkLBHaproxyMaxConn.key()); |
| } else { |
| maxconn = offering.getConcurrentConnections().toString(); |
| } |
| final LoadBalancerConfigCommand cmd = |
| new LoadBalancerConfigCommand(lbs, guestNic.getIPv4Address(), guestNic.getIPv4Address(), internalLbVm.getPrivateIpAddress(), _itMgr.toNicTO(guestNicProfile, |
| internalLbVm.getHypervisorType()), internalLbVm.getVpcId(), maxconn, offering.isKeepAliveEnabled()); |
| |
| cmd.lbStatsVisibility = _configDao.getValue(Config.NetworkLBHaproxyStatsVisbility.key()); |
| cmd.lbStatsUri = _configDao.getValue(Config.NetworkLBHaproxyStatsUri.key()); |
| cmd.lbStatsAuth = _configDao.getValue(Config.NetworkLBHaproxyStatsAuth.key()); |
| cmd.lbStatsPort = _configDao.getValue(Config.NetworkLBHaproxyStatsPort.key()); |
| |
| cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, getInternalLbControlIp(internalLbVm.getId())); |
| cmd.setAccessDetail(NetworkElementCommand.ROUTER_GUEST_IP, guestNic.getIPv4Address()); |
| cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, internalLbVm.getInstanceName()); |
| final DataCenterVO dcVo = _dcDao.findById(internalLbVm.getDataCenterId()); |
| cmd.setAccessDetail(NetworkElementCommand.ZONE_NETWORK_TYPE, dcVo.getNetworkType().toString()); |
| cmds.addCommand(cmd); |
| } |
| |
| protected String getInternalLbControlIp(final long internalLbVmId) { |
| String controlIpAddress = null; |
| final List<NicVO> nics = _nicDao.listByVmId(internalLbVmId); |
| for (final NicVO nic : nics) { |
| final Network ntwk = _ntwkModel.getNetwork(nic.getNetworkId()); |
| if (ntwk.getTrafficType() == TrafficType.Control) { |
| controlIpAddress = nic.getIPv4Address(); |
| } |
| } |
| |
| if (controlIpAddress == null) { |
| logger.warn("Unable to find Internal LB control ip in its attached NICs!. Internal LB vm: " + internalLbVmId); |
| final DomainRouterVO internalLbVm = _internalLbVmDao.findById(internalLbVmId); |
| return internalLbVm.getPrivateIpAddress(); |
| } |
| |
| return controlIpAddress; |
| } |
| |
| @Override |
| public boolean destroyInternalLbVm(final long vmId, final Account caller, final Long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException { |
| if (logger.isDebugEnabled()) { |
| logger.debug("Attempting to destroy Internal LB vm " + vmId); |
| } |
| |
| final DomainRouterVO internalLbVm = _internalLbVmDao.findById(vmId); |
| if (internalLbVm == null) { |
| return true; |
| } |
| |
| _accountMgr.checkAccess(caller, null, true, internalLbVm); |
| |
| _itMgr.expunge(internalLbVm.getUuid()); |
| _internalLbVmDao.remove(internalLbVm.getId()); |
| return true; |
| } |
| |
| @Override |
| public VirtualRouter stopInternalLbVm(final long vmId, final boolean forced, final Account caller, final long callerUserId) throws ConcurrentOperationException, ResourceUnavailableException { |
| final DomainRouterVO internalLbVm = _internalLbVmDao.findById(vmId); |
| if (internalLbVm == null || internalLbVm.getRole() != Role.INTERNAL_LB_VM) { |
| throw new InvalidParameterValueException("Can't find internal lb vm by id specified"); |
| } |
| |
| //check permissions |
| _accountMgr.checkAccess(caller, null, true, internalLbVm); |
| |
| return stopInternalLbVm(internalLbVm, forced, caller, callerUserId); |
| } |
| |
| protected VirtualRouter stopInternalLbVm(final DomainRouterVO internalLbVm, final boolean forced, final Account caller, final long callerUserId) throws ResourceUnavailableException, |
| ConcurrentOperationException { |
| logger.debug("Stopping internal lb vm " + internalLbVm); |
| try { |
| _itMgr.advanceStop(internalLbVm.getUuid(), forced); |
| return _internalLbVmDao.findById(internalLbVm.getId()); |
| } catch (final OperationTimedoutException e) { |
| throw new CloudRuntimeException("Unable to stop " + internalLbVm, e); |
| } |
| } |
| |
| @Override |
| public List<DomainRouterVO> deployInternalLbVm(final Network guestNetwork, final Ip requestedGuestIp, final DeployDestination dest, final Account owner, final Map<Param, Object> params) |
| throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { |
| |
| final List<DomainRouterVO> internalLbVms = findOrDeployInternalLbVm(guestNetwork, requestedGuestIp, dest, owner, params); |
| |
| return startInternalLbVms(params, internalLbVms); |
| } |
| |
| protected List<DomainRouterVO> startInternalLbVms(final Map<Param, Object> params, final List<DomainRouterVO> internalLbVms) throws StorageUnavailableException, |
| InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { |
| List<DomainRouterVO> runningInternalLbVms = null; |
| |
| if (internalLbVms != null) { |
| runningInternalLbVms = new ArrayList<DomainRouterVO>(); |
| } else { |
| logger.debug("Have no internal lb vms to start"); |
| return null; |
| } |
| |
| for (DomainRouterVO internalLbVm : internalLbVms) { |
| if (internalLbVm.getState() != VirtualMachine.State.Running) { |
| internalLbVm = startInternalLbVm(internalLbVm, _accountMgr.getSystemAccount(), User.UID_SYSTEM, params); |
| } |
| |
| if (internalLbVm != null) { |
| runningInternalLbVms.add(internalLbVm); |
| } |
| } |
| return runningInternalLbVms; |
| } |
| |
| @DB |
| protected List<DomainRouterVO> findOrDeployInternalLbVm(final Network guestNetwork, final Ip requestedGuestIp, final DeployDestination dest, final Account owner, final Map<Param, Object> params) |
| throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException { |
| |
| List<DomainRouterVO> internalLbVms = new ArrayList<DomainRouterVO>(); |
| final Network lock = _networkDao.acquireInLockTable(guestNetwork.getId(), NetworkOrchestrationService.NetworkLockTimeout.value()); |
| if (lock == null) { |
| throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); |
| } |
| |
| if (logger.isDebugEnabled()) { |
| logger.debug("Lock is acquired for network id " + lock.getId() + " as a part of internal lb startup in " + dest); |
| } |
| |
| final long internalLbProviderId = getInternalLbProviderId(guestNetwork); |
| |
| try { |
| assert guestNetwork.getState() == Network.State.Implemented || guestNetwork.getState() == Network.State.Setup || |
| guestNetwork.getState() == Network.State.Implementing : "Network is not yet fully implemented: " + guestNetwork; |
| assert guestNetwork.getTrafficType() == TrafficType.Guest; |
| |
| //deploy internal lb vm |
| final Pair<DeploymentPlan, List<DomainRouterVO>> planAndInternalLbVms = getDeploymentPlanAndInternalLbVms(dest, guestNetwork.getId(), requestedGuestIp); |
| internalLbVms = planAndInternalLbVms.second(); |
| final DeploymentPlan plan = planAndInternalLbVms.first(); |
| |
| if (internalLbVms.size() > 0) { |
| logger.debug("Found " + internalLbVms.size() + " internal lb vms for the requested IP " + requestedGuestIp.addr()); |
| return internalLbVms; |
| } |
| |
| final LinkedHashMap<Network, List<? extends NicProfile>> networks = createInternalLbVmNetworks(guestNetwork, plan, requestedGuestIp); |
| long internalLbVmOfferingId = _internalLbVmOfferingId; |
| if (internalLbVmOfferingId == 0L) { |
| ServiceOfferingVO serviceOffering = _serviceOfferingDao.findDefaultSystemOffering(ServiceOffering.internalLbVmDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dest.getDataCenter().getId())); |
| internalLbVmOfferingId = serviceOffering.getId(); |
| } |
| //Pass startVm=false as we are holding the network lock that needs to be released at the end of vm allocation |
| final DomainRouterVO internalLbVm = |
| deployInternalLbVm(owner, dest, plan, params, internalLbProviderId, internalLbVmOfferingId, guestNetwork.getVpcId(), networks, false); |
| if (internalLbVm != null) { |
| _internalLbVmDao.addRouterToGuestNetwork(internalLbVm, guestNetwork); |
| internalLbVms.add(internalLbVm); |
| } |
| } finally { |
| if (lock != null) { |
| _networkDao.releaseFromLockTable(lock.getId()); |
| if (logger.isDebugEnabled()) { |
| logger.debug("Lock is released for network id " + lock.getId() + " as a part of internal lb vm startup in " + dest); |
| } |
| } |
| } |
| return internalLbVms; |
| } |
| |
| protected long getInternalLbProviderId(final Network guestNetwork) { |
| final Type type = Type.InternalLbVm; |
| final long physicalNetworkId = _ntwkModel.getPhysicalNetworkId(guestNetwork); |
| |
| final PhysicalNetworkServiceProvider provider = _physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); |
| if (provider == null) { |
| throw new CloudRuntimeException("Cannot find service provider " + type.toString() + " in physical network " + physicalNetworkId); |
| } |
| |
| final VirtualRouterProvider internalLbProvider = _vrProviderDao.findByNspIdAndType(provider.getId(), type); |
| if (internalLbProvider == null) { |
| throw new CloudRuntimeException("Cannot find provider " + type.toString() + " as service provider " + provider.getId()); |
| } |
| |
| return internalLbProvider.getId(); |
| } |
| |
| protected LinkedHashMap<Network, List<? extends NicProfile>> createInternalLbVmNetworks(final Network guestNetwork, final DeploymentPlan plan, final Ip guestIp) throws ConcurrentOperationException, |
| InsufficientAddressCapacityException { |
| |
| //Form networks |
| final LinkedHashMap<Network, List<? extends NicProfile>> networks = new LinkedHashMap<Network, List<? extends NicProfile>>(3); |
| |
| //1) Guest network - default |
| if (guestNetwork != null) { |
| logger.debug("Adding nic for Internal LB in Guest network " + guestNetwork); |
| final NicProfile guestNic = new NicProfile(); |
| if (guestIp != null) { |
| guestNic.setIPv4Address(guestIp.addr()); |
| } else { |
| guestNic.setIPv4Address(_ipAddrMgr.acquireGuestIpAddress(guestNetwork, null)); |
| } |
| guestNic.setIPv4Gateway(guestNetwork.getGateway()); |
| guestNic.setBroadcastUri(guestNetwork.getBroadcastUri()); |
| guestNic.setBroadcastType(guestNetwork.getBroadcastDomainType()); |
| guestNic.setIsolationUri(guestNetwork.getBroadcastUri()); |
| guestNic.setMode(guestNetwork.getMode()); |
| final String gatewayCidr = guestNetwork.getCidr(); |
| guestNic.setIPv4Netmask(NetUtils.getCidrNetmask(gatewayCidr)); |
| guestNic.setDefaultNic(true); |
| networks.put(guestNetwork, new ArrayList<NicProfile>(Arrays.asList(guestNic))); |
| } |
| |
| //2) Control network |
| logger.debug("Adding nic for Internal LB vm in Control network "); |
| final List<? extends NetworkOffering> offerings = _ntwkModel.getSystemAccountNetworkOfferings(NetworkOffering.SystemControlNetwork); |
| final NetworkOffering controlOffering = offerings.get(0); |
| final Network controlConfig = _ntwkMgr.setupNetwork(_accountMgr.getSystemAccount(), controlOffering, plan, null, null, false).get(0); |
| networks.put(controlConfig, new ArrayList<NicProfile>()); |
| |
| return networks; |
| } |
| |
| protected Pair<DeploymentPlan, List<DomainRouterVO>> getDeploymentPlanAndInternalLbVms(final DeployDestination dest, final long guestNetworkId, final Ip requestedGuestIp) { |
| final long dcId = dest.getDataCenter().getId(); |
| final DeploymentPlan plan = new DataCenterDeployment(dcId); |
| final List<DomainRouterVO> internalLbVms = findInternalLbVms(guestNetworkId, requestedGuestIp); |
| |
| return new Pair<DeploymentPlan, List<DomainRouterVO>>(plan, internalLbVms); |
| |
| } |
| |
| @Override |
| public List<DomainRouterVO> findInternalLbVms(final long guestNetworkId, final Ip requestedGuestIp) { |
| final List<DomainRouterVO> internalLbVms = _internalLbVmDao.listByNetworkAndRole(guestNetworkId, Role.INTERNAL_LB_VM); |
| if (requestedGuestIp != null && !internalLbVms.isEmpty()) { |
| final Iterator<DomainRouterVO> it = internalLbVms.iterator(); |
| while (it.hasNext()) { |
| final DomainRouterVO vm = it.next(); |
| final Nic nic = _nicDao.findByNtwkIdAndInstanceId(guestNetworkId, vm.getId()); |
| if (!nic.getIPv4Address().equalsIgnoreCase(requestedGuestIp.addr())) { |
| it.remove(); |
| } |
| } |
| } |
| return internalLbVms; |
| } |
| |
| protected DomainRouterVO deployInternalLbVm(final Account owner, final DeployDestination dest, final DeploymentPlan plan, final Map<Param, Object> params, final long internalLbProviderId, |
| final long svcOffId, final Long vpcId, final LinkedHashMap<Network, List<? extends NicProfile>> networks, final boolean startVm) throws ConcurrentOperationException, |
| InsufficientAddressCapacityException, InsufficientServerCapacityException, InsufficientCapacityException, StorageUnavailableException, |
| ResourceUnavailableException { |
| |
| final ServiceOfferingVO routerOffering = _serviceOfferingDao.findById(svcOffId); |
| |
| // Internal lb is the network element, we don't know the hypervisor type yet. |
| // Try to allocate the internal lb twice using diff hypervisors, and when failed both times, throw the exception up |
| final List<HypervisorType> hypervisors = getHypervisors(dest, plan, null); |
| |
| int allocateRetry = 0; |
| int startRetry = 0; |
| DomainRouterVO internalLbVm = null; |
| for (final Iterator<HypervisorType> iter = hypervisors.iterator(); iter.hasNext();) { |
| final HypervisorType hType = iter.next(); |
| try { |
| final long id = _internalLbVmDao.getNextInSequence(Long.class, "id"); |
| if (logger.isDebugEnabled()) { |
| logger.debug("Creating the internal lb vm " + id + " in datacenter " + dest.getDataCenter() + " with hypervisor type " + hType); |
| } |
| String templateName = null; |
| if (hType.equals(XenServer)) { |
| templateName = VirtualNetworkApplianceManager.RouterTemplateXen.valueIn(dest.getDataCenter().getId()); |
| } else if (hType.equals(KVM)) { |
| templateName = VirtualNetworkApplianceManager.RouterTemplateKvm.valueIn(dest.getDataCenter().getId()); |
| } else if (hType.equals(VMware)) { |
| templateName = VirtualNetworkApplianceManager.RouterTemplateVmware.valueIn(dest.getDataCenter().getId()); |
| } else if (hType.equals(Hyperv)) { |
| templateName = VirtualNetworkApplianceManager.RouterTemplateHyperV.valueIn(dest.getDataCenter().getId()); |
| } else if (hType.equals(LXC)) { |
| templateName = VirtualNetworkApplianceManager.RouterTemplateLxc.valueIn(dest.getDataCenter().getId()); |
| } |
| final VMTemplateVO template = _templateDao.findRoutingTemplate(hType, templateName); |
| |
| if (template == null) { |
| logger.debug(hType + " won't support system vm, skip it"); |
| continue; |
| } |
| |
| long userId = CallContext.current().getCallingUserId(); |
| if (CallContext.current().getCallingAccount().getId() != owner.getId()) { |
| List<UserVO> userVOs = _userDao.listByAccount(owner.getAccountId()); |
| if (!userVOs.isEmpty()) { |
| userId = userVOs.get(0).getId(); |
| } |
| } |
| |
| internalLbVm = |
| new DomainRouterVO(id, routerOffering.getId(), internalLbProviderId, VirtualMachineName.getSystemVmName(id, _instance, InternalLbVmNamePrefix), |
| template.getId(), template.getHypervisorType(), template.getGuestOSId(), owner.getDomainId(), owner.getId(), userId, false, RedundantState.UNKNOWN, false, false, VirtualMachine.Type.InternalLoadBalancerVm, vpcId); |
| internalLbVm.setRole(Role.INTERNAL_LB_VM); |
| internalLbVm = _internalLbVmDao.persist(internalLbVm); |
| _itMgr.allocate(internalLbVm.getInstanceName(), template, routerOffering, networks, plan, null); |
| internalLbVm = _internalLbVmDao.findById(internalLbVm.getId()); |
| } catch (final InsufficientCapacityException ex) { |
| if (allocateRetry < 2 && iter.hasNext()) { |
| logger.debug("Failed to allocate the Internal lb vm with hypervisor type " + hType + ", retrying one more time"); |
| continue; |
| } else { |
| throw ex; |
| } |
| } finally { |
| allocateRetry++; |
| } |
| |
| if (startVm) { |
| try { |
| internalLbVm = startInternalLbVm(internalLbVm, _accountMgr.getSystemAccount(), User.UID_SYSTEM, params); |
| break; |
| } catch (final InsufficientCapacityException ex) { |
| if (startRetry < 2 && iter.hasNext()) { |
| logger.debug("Failed to start the Internal lb vm " + internalLbVm + " with hypervisor type " + hType + ", " + |
| "destroying it and recreating one more time"); |
| // destroy the internal lb vm |
| destroyInternalLbVm(internalLbVm.getId(), _accountMgr.getSystemAccount(), User.UID_SYSTEM); |
| continue; |
| } else { |
| throw ex; |
| } |
| } finally { |
| startRetry++; |
| } |
| } else { |
| //return stopped internal lb vm |
| return internalLbVm; |
| } |
| } |
| return internalLbVm; |
| } |
| |
| protected DomainRouterVO startInternalLbVm(DomainRouterVO internalLbVm, final Account caller, final long callerUserId, final Map<Param, Object> params) |
| throws StorageUnavailableException, InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { |
| logger.debug("Starting Internal LB VM " + internalLbVm); |
| _itMgr.start(internalLbVm.getUuid(), params, null, null); |
| if (internalLbVm.isStopPending()) { |
| logger.info("Clear the stop pending flag of Internal LB VM " + internalLbVm.getHostName() + " after start router successfully!"); |
| internalLbVm.setStopPending(false); |
| internalLbVm = _internalLbVmDao.persist(internalLbVm); |
| } |
| return _internalLbVmDao.findById(internalLbVm.getId()); |
| } |
| |
| protected List<HypervisorType> getHypervisors(final DeployDestination dest, final DeploymentPlan plan, final List<HypervisorType> supportedHypervisors) |
| throws InsufficientServerCapacityException { |
| List<HypervisorType> hypervisors = new ArrayList<HypervisorType>(); |
| |
| final HypervisorType defaults = _resourceMgr.getDefaultHypervisor(dest.getDataCenter().getId()); |
| if (defaults != HypervisorType.None) { |
| hypervisors.add(defaults); |
| } else { |
| //if there is no default hypervisor, get it from the cluster |
| hypervisors = _resourceMgr.getSupportedHypervisorTypes(dest.getDataCenter().getId(), true, plan.getPodId()); |
| } |
| |
| //keep only elements defined in supported hypervisors |
| final StringBuilder hTypesStr = new StringBuilder(); |
| if (supportedHypervisors != null && !supportedHypervisors.isEmpty()) { |
| hypervisors.retainAll(supportedHypervisors); |
| for (final HypervisorType hType : supportedHypervisors) { |
| hTypesStr.append(hType).append(" "); |
| } |
| } |
| |
| if (hypervisors.isEmpty()) { |
| throw new InsufficientServerCapacityException("Unable to create internal lb vm, " + "there are no clusters in the zone ", DataCenter.class, |
| dest.getDataCenter().getId()); |
| } |
| return hypervisors; |
| } |
| |
| @Override |
| public boolean applyLoadBalancingRules(final Network network, final List<LoadBalancingRule> rules, final List<? extends VirtualRouter> internalLbVms) |
| throws ResourceUnavailableException { |
| if (rules == null || rules.isEmpty()) { |
| logger.debug("No lb rules to be applied for network " + network); |
| return true; |
| } |
| logger.info("lb rules to be applied for network "); |
| //only one internal lb vm is supported per ip address at this time |
| if (internalLbVms == null || internalLbVms.isEmpty()) { |
| throw new CloudRuntimeException("Can't apply the lb rules on network " + network + " as the list of internal lb vms is empty"); |
| } |
| |
| final VirtualRouter lbVm = internalLbVms.get(0); |
| if (lbVm.getState() == State.Running) { |
| return sendLBRules(lbVm, rules, network.getId()); |
| } else if (lbVm.getState() == State.Stopped || lbVm.getState() == State.Stopping) { |
| logger.debug("Internal LB VM " + lbVm.getInstanceName() + " is in " + lbVm.getState() + ", so not sending apply lb rules commands to the backend"); |
| return true; |
| } else { |
| logger.warn("Unable to apply lb rules, Internal LB VM is not in the right state " + lbVm.getState()); |
| throw new ResourceUnavailableException("Unable to apply lb rules; Internal LB VM is not in the right state", DataCenter.class, lbVm.getDataCenterId()); |
| } |
| } |
| |
| protected boolean sendLBRules(final VirtualRouter internalLbVm, final List<LoadBalancingRule> rules, final long guestNetworkId) throws ResourceUnavailableException { |
| final Commands cmds = new Commands(Command.OnError.Continue); |
| createApplyLoadBalancingRulesCommands(rules, internalLbVm, cmds, guestNetworkId); |
| return sendCommandsToInternalLbVm(internalLbVm, cmds); |
| } |
| |
| protected boolean sendCommandsToInternalLbVm(final VirtualRouter internalLbVm, final Commands cmds) throws AgentUnavailableException { |
| Answer[] answers = null; |
| try { |
| answers = _agentMgr.send(internalLbVm.getHostId(), cmds); |
| } catch (final OperationTimedoutException e) { |
| logger.warn("Timed Out", e); |
| throw new AgentUnavailableException("Unable to send commands to virtual router ", internalLbVm.getHostId(), e); |
| } |
| |
| if (answers == null) { |
| return false; |
| } |
| |
| if (answers.length != cmds.size()) { |
| return false; |
| } |
| |
| boolean result = true; |
| if (answers.length > 0) { |
| for (final Answer answer : answers) { |
| if (!answer.getResult()) { |
| result = false; |
| break; |
| } |
| } |
| } |
| return result; |
| } |
| |
| @Override |
| public VirtualRouter startInternalLbVm(final long internalLbVmId, final Account caller, final long callerUserId) throws StorageUnavailableException, InsufficientCapacityException, |
| ConcurrentOperationException, ResourceUnavailableException { |
| |
| final DomainRouterVO internalLbVm = _internalLbVmDao.findById(internalLbVmId); |
| if (internalLbVm == null || internalLbVm.getRole() != Role.INTERNAL_LB_VM) { |
| throw new InvalidParameterValueException("Can't find internal lb vm by id specified"); |
| } |
| |
| //check permissions |
| _accountMgr.checkAccess(caller, null, true, internalLbVm); |
| |
| return startInternalLbVm(internalLbVm, caller, callerUserId, null); |
| } |
| } |
