| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| package org.apache.cloudstack.network.router.deployment; |
| |
| import java.util.ArrayList; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Objects; |
| |
| import com.cloud.dc.DataCenter; |
| import com.cloud.dc.Vlan; |
| import com.cloud.network.dao.NetworkDetailVO; |
| import com.cloud.network.dao.NetworkDetailsDao; |
| import com.cloud.network.dao.NsxProviderDao; |
| import com.cloud.network.element.NsxProviderVO; |
| import com.cloud.network.router.VirtualRouter; |
| import com.cloud.storage.DiskOfferingVO; |
| import com.cloud.storage.dao.DiskOfferingDao; |
| import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; |
| import org.apache.logging.log4j.Logger; |
| import org.apache.logging.log4j.LogManager; |
| |
| import com.cloud.configuration.ConfigurationManagerImpl; |
| import com.cloud.dc.DataCenter.NetworkType; |
| import com.cloud.dc.HostPodVO; |
| import com.cloud.dc.Pod; |
| import com.cloud.dc.dao.HostPodDao; |
| import com.cloud.deploy.DataCenterDeployment; |
| import com.cloud.deploy.DeployDestination; |
| import com.cloud.deploy.DeploymentPlan; |
| import com.cloud.exception.ConcurrentOperationException; |
| import com.cloud.exception.InsufficientAddressCapacityException; |
| import com.cloud.exception.InsufficientCapacityException; |
| import com.cloud.exception.ResourceUnavailableException; |
| import com.cloud.network.IpAddressManager; |
| import com.cloud.network.Network; |
| import com.cloud.network.Network.Provider; |
| import com.cloud.network.Network.Service; |
| import com.cloud.network.NetworkModel; |
| import com.cloud.network.Networks.TrafficType; |
| import com.cloud.network.PhysicalNetworkServiceProvider; |
| import com.cloud.network.VirtualRouterProvider; |
| import com.cloud.network.VirtualRouterProvider.Type; |
| import com.cloud.network.addr.PublicIp; |
| import com.cloud.network.dao.IPAddressDao; |
| import com.cloud.network.dao.NetworkDao; |
| import com.cloud.network.dao.PhysicalNetworkServiceProviderDao; |
| import com.cloud.network.dao.UserIpv6AddressDao; |
| import com.cloud.network.dao.VirtualRouterProviderDao; |
| import com.cloud.network.router.NetworkHelper; |
| import com.cloud.network.router.VirtualNetworkApplianceManager; |
| import com.cloud.network.router.VirtualRouter.Role; |
| import com.cloud.network.vpc.Vpc; |
| import com.cloud.offering.ServiceOffering; |
| import com.cloud.offerings.dao.NetworkOfferingDao; |
| import com.cloud.service.ServiceOfferingVO; |
| import com.cloud.service.dao.ServiceOfferingDao; |
| import com.cloud.user.Account; |
| import com.cloud.user.AccountManager; |
| import com.cloud.utils.db.DB; |
| import com.cloud.utils.db.JoinBuilder; |
| import com.cloud.utils.db.SearchBuilder; |
| import com.cloud.utils.db.SearchCriteria; |
| import com.cloud.utils.exception.CloudRuntimeException; |
| import com.cloud.vm.DomainRouterVO; |
| import com.cloud.vm.VMInstanceVO; |
| import com.cloud.vm.VirtualMachine; |
| import com.cloud.vm.VirtualMachineProfile.Param; |
| import com.cloud.vm.dao.DomainRouterDao; |
| import com.cloud.vm.dao.NicDao; |
| import com.cloud.vm.dao.VMInstanceDao; |
| |
| public class RouterDeploymentDefinition { |
| protected Logger logger = LogManager.getLogger(getClass()); |
| |
| protected static final int LIMIT_NUMBER_OF_ROUTERS = 5; |
| protected static final int MAX_NUMBER_OF_ROUTERS = 2; |
| |
| protected NetworkDao networkDao; |
| protected DomainRouterDao routerDao; |
| protected NsxProviderDao nsxProviderDao; |
| protected PhysicalNetworkServiceProviderDao physicalProviderDao; |
| protected NetworkModel networkModel; |
| protected VirtualRouterProviderDao vrProviderDao; |
| protected NetworkOfferingDao networkOfferingDao; |
| protected ServiceOfferingDao serviceOfferingDao; |
| protected DiskOfferingDao diskOfferingDao; |
| protected IpAddressManager ipAddrMgr; |
| protected VMInstanceDao vmDao; |
| protected HostPodDao podDao; |
| protected AccountManager accountMgr; |
| protected NetworkOrchestrationService networkMgr; |
| protected NicDao nicDao; |
| protected UserIpv6AddressDao ipv6Dao; |
| protected IPAddressDao ipAddressDao; |
| protected VirtualRouterProvider vrProvider; |
| protected NetworkHelper nwHelper; |
| |
| protected Network guestNetwork; |
| protected DeployDestination dest; |
| protected Account owner; |
| protected Map<Param, Object> params; |
| protected DeploymentPlan plan; |
| protected List<DomainRouterVO> routers = new ArrayList<>(); |
| protected Long serviceOfferingId; |
| protected Long tableLockId; |
| protected boolean isPublicNetwork; |
| protected PublicIp sourceNatIp; |
| protected NetworkDetailsDao networkDetailsDao; |
| |
| protected RouterDeploymentDefinition(final Network guestNetwork, final DeployDestination dest, |
| final Account owner, final Map<Param, Object> params) { |
| |
| this.guestNetwork = guestNetwork; |
| this.dest = dest; |
| this.owner = owner; |
| this.params = params; |
| } |
| |
| public Long getServiceOfferingId() { |
| return serviceOfferingId; |
| } |
| |
| public Vpc getVpc() { |
| return null; |
| } |
| |
| public Network getGuestNetwork() { |
| return guestNetwork; |
| } |
| |
| public DeployDestination getDest() { |
| return dest; |
| } |
| |
| public Account getOwner() { |
| return owner; |
| } |
| |
| public Map<Param, Object> getParams() { |
| return params; |
| } |
| |
| public boolean isRedundant() { |
| return guestNetwork.isRedundant(); |
| } |
| |
| public boolean isRollingRestart() { |
| return guestNetwork.isRollingRestart(); |
| } |
| |
| public DeploymentPlan getPlan() { |
| return plan; |
| } |
| |
| public boolean isVpcRouter() { |
| return false; |
| } |
| |
| public Pod getPod() { |
| return dest.getPod(); |
| } |
| |
| public Long getPodId() { |
| return dest.getPod() == null ? null : dest.getPod().getId(); |
| } |
| |
| public List<DomainRouterVO> getRouters() { |
| return routers; |
| } |
| |
| public VirtualRouterProvider getVirtualProvider() { |
| return vrProvider; |
| } |
| |
| public boolean isBasic() { |
| return dest.getDataCenter().getNetworkType() == NetworkType.Basic; |
| } |
| |
| public boolean isPublicNetwork() { |
| return isPublicNetwork; |
| } |
| |
| public PublicIp getSourceNatIP() { |
| return sourceNatIp; |
| } |
| |
| protected void generateDeploymentPlan() { |
| final long dcId = dest.getDataCenter().getId(); |
| Long podId = null; |
| if (isBasic()) { |
| if (dest.getPod() == null) { |
| throw new CloudRuntimeException("Pod id is expected in deployment destination"); |
| } |
| podId = dest.getPod().getId(); |
| } |
| plan = new DataCenterDeployment(dcId, podId, null, null, null, null); |
| } |
| |
| public List<DomainRouterVO> deployVirtualRouter() throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { |
| findOrDeployVirtualRouter(); |
| return nwHelper.startRouters(this); |
| } |
| |
| private boolean isRouterDeployed() throws ResourceUnavailableException { |
| boolean isDeployed = true; |
| checkPreconditions(); |
| final List<DeployDestination> destinations = findDestinations(); |
| for (final DeployDestination destination : destinations) { |
| dest = destination; |
| generateDeploymentPlan(); |
| planDeploymentRouters(); |
| if (getNumberOfRoutersToDeploy() > 0 && prepareDeployment()) { |
| isDeployed = false; |
| break; |
| } |
| } |
| return isDeployed; |
| } |
| |
| @DB |
| protected void findOrDeployVirtualRouter() throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException { |
| if (!isRouterDeployed()) { |
| try { |
| lock(); |
| // reset router list that got populated during isRouterDeployed() |
| routers.clear(); |
| checkPreconditions(); |
| |
| // dest has pod=null, for Basic Zone findOrDeployVRs for all Pods |
| final List<DeployDestination> destinations = findDestinations(); |
| for (final DeployDestination destination : destinations) { |
| dest = destination; |
| generateDeploymentPlan(); |
| executeDeployment(); |
| } |
| } finally { |
| unlock(); |
| } |
| } |
| } |
| |
| protected void lock() { |
| final Network lock = networkDao.acquireInLockTable(guestNetwork.getId(), NetworkOrchestrationService.NetworkLockTimeout.value()); |
| if (lock == null) { |
| throw new ConcurrentOperationException("Unable to lock network " + guestNetwork.getId()); |
| } |
| tableLockId = lock.getId(); |
| } |
| |
| protected void unlock() { |
| if (tableLockId != null) { |
| networkDao.releaseFromLockTable(tableLockId); |
| if (logger.isDebugEnabled()) { |
| logger.debug("Lock is released for network id " + tableLockId + " as a part of router startup in " + dest); |
| } |
| } |
| } |
| |
| protected void checkPreconditions() throws ResourceUnavailableException { |
| if (guestNetwork.getState() != Network.State.Implemented && guestNetwork.getState() != Network.State.Setup && guestNetwork.getState() != Network.State.Implementing) { |
| throw new ResourceUnavailableException("Network is not yet fully implemented: " + guestNetwork, Network.class, guestNetwork.getId()); |
| } |
| |
| if (guestNetwork.getTrafficType() != TrafficType.Guest) { |
| throw new ResourceUnavailableException("Network is not type Guest as expected: " + guestNetwork, Network.class, guestNetwork.getId()); |
| } |
| } |
| |
| protected List<DeployDestination> findDestinations() { |
| // dest has pod=null, for Basic Zone findOrDeployVRs for all Pods |
| final List<DeployDestination> destinations = new ArrayList<DeployDestination>(); |
| |
| // for basic zone, if 'dest' has pod set to null then this is network |
| // restart scenario otherwise it is a vm deployment scenario |
| if (isBasic() && dest.getPod() == null) { |
| // Find all pods in the data center with running or starting user vms |
| final long dcId = dest.getDataCenter().getId(); |
| final List<HostPodVO> pods = listByDataCenterIdVMTypeAndStates(dcId, VirtualMachine.Type.User, VirtualMachine.State.Starting, VirtualMachine.State.Running); |
| |
| // Loop through all the pods skip those with running or starting VRs |
| for (final HostPodVO pod : pods) { |
| // Get list of VRs in starting or running state |
| final long podId = pod.getId(); |
| final List<DomainRouterVO> virtualRouters = routerDao.listByPodIdAndStates(podId, VirtualMachine.State.Starting, VirtualMachine.State.Running); |
| |
| if (virtualRouters.size() > 1) { |
| // FIXME Find or create a better and more specific exception for this |
| throw new CloudRuntimeException("Pod can have utmost one VR in Basic Zone, please check!"); |
| } |
| |
| // Add virtualRouters to the routers, this avoids the situation when |
| // all routers are skipped and VirtualRouterElement throws exception |
| routers.addAll(virtualRouters); |
| |
| // If List size is one, we already have a starting or running VR, skip deployment |
| if (virtualRouters.size() == 1) { |
| logger.debug("Skipping VR deployment: Found a running or starting VR in Pod " + pod.getName() + " id=" + podId); |
| continue; |
| } |
| // Add new DeployDestination for this pod |
| destinations.add(new DeployDestination(dest.getDataCenter(), pod, null, null)); |
| } |
| } else { |
| // Else, just add the supplied dest |
| destinations.add(dest); |
| } |
| return destinations; |
| } |
| |
| protected int getNumberOfRoutersToDeploy() { |
| // TODO Are we sure this makes sense? Somebody said 5 was too many? |
| if (routers.size() >= LIMIT_NUMBER_OF_ROUTERS) { |
| logger.error("Too many redundant routers!"); |
| } |
| |
| // If old network is redundant but new is single router, then |
| // routers.size() = 2 but routerCount = 1 |
| int routersExpected = 1; |
| if (isRedundant() || isRollingRestart()) { |
| routersExpected = 2; |
| } |
| return routersExpected < routers.size() ? 0 : routersExpected - routers.size(); |
| } |
| |
| protected void setupAccountOwner() { |
| if (networkModel.isNetworkSystem(guestNetwork) || guestNetwork.getGuestType() == Network.GuestType.Shared) { |
| owner = accountMgr.getAccount(Account.ACCOUNT_ID_SYSTEM); |
| } |
| } |
| |
| /** |
| * It executes last pending tasks to prepare the deployment and checks the |
| * deployment can proceed. If it can't it return false |
| * |
| * @return if the deployment can proceed |
| */ |
| protected boolean prepareDeployment() { |
| setupAccountOwner(); |
| |
| // Check if public network has to be set on VR |
| isPublicNetwork = networkModel.isProviderSupportServiceInNetwork(guestNetwork.getId(), Service.SourceNat, Provider.VirtualRouter); |
| |
| boolean canProceed = true; |
| if (isRedundant() && !isPublicNetwork) { |
| // TODO Shouldn't be this throw an exception instead of log error and empty list of routers |
| logger.error("Didn't support redundant virtual router without public network!"); |
| routers = new ArrayList<DomainRouterVO>(); |
| canProceed = false; |
| } |
| |
| return canProceed; |
| } |
| |
| /** |
| * Executes preparation and deployment of the routers. After this method |
| * ends, {@link this#routers} should have all of the deployed routers ready |
| * for start, and no more. |
| * |
| * @throws ConcurrentOperationException |
| * @throws InsufficientCapacityException |
| * @throws ResourceUnavailableException |
| */ |
| protected void executeDeployment() |
| throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException { |
| //Check current redundant routers, if possible(all routers are stopped), reset the priority |
| planDeploymentRouters(); |
| |
| if (getNumberOfRoutersToDeploy() > 0 && prepareDeployment()) { |
| findVirtualProvider(); |
| findServiceOfferingId(); |
| findSourceNatIP(); |
| deployAllVirtualRouters(); |
| } |
| } |
| |
| protected void findSourceNatIP() throws InsufficientAddressCapacityException, ConcurrentOperationException { |
| sourceNatIp = null; |
| DataCenter zone = dest.getDataCenter(); |
| Long zoneId = null; |
| if (Objects.nonNull(zone)) { |
| zoneId = zone.getId(); |
| } |
| NsxProviderVO nsxProvider = nsxProviderDao.findByZoneId(zoneId); |
| |
| if (isPublicNetwork) { |
| if (Objects.isNull(nsxProvider)) { |
| sourceNatIp = ipAddrMgr.assignSourceNatIpAddressToGuestNetwork(owner, guestNetwork); |
| } else { |
| sourceNatIp = ipAddrMgr.assignPublicIpAddress(zoneId, getPodId(), owner, Vlan.VlanType.VirtualNetwork, null, null, false, true); |
| } |
| } |
| } |
| |
| protected void findDefaultServiceOfferingId() { |
| ServiceOfferingVO serviceOffering = serviceOfferingDao.findDefaultSystemOffering(ServiceOffering.routerDefaultOffUniqueName, ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dest.getDataCenter().getId())); |
| serviceOfferingId = serviceOffering.getId(); |
| } |
| |
| protected void findAccountServiceOfferingId(long accountId) { |
| String accountRouterOffering = VirtualNetworkApplianceManager.VirtualRouterServiceOffering.valueIn(accountId); |
| String globalRouterOffering = VirtualNetworkApplianceManager.VirtualRouterServiceOffering.value(); |
| if (accountRouterOffering != null) { |
| verifyServiceOfferingByUuid(accountRouterOffering); |
| } |
| if (serviceOfferingId == null && globalRouterOffering != accountRouterOffering) { |
| verifyServiceOfferingByUuid(globalRouterOffering); |
| } |
| } |
| |
| private void verifyServiceOfferingByUuid(String offeringUuid) { |
| logger.debug("Verifying router service offering with uuid : " + offeringUuid); |
| ServiceOfferingVO serviceOffering = serviceOfferingDao.findByUuid(offeringUuid); |
| if (serviceOffering != null && serviceOffering.isSystemUse() && ServiceOffering.State.Active.equals(serviceOffering.getState())) { |
| DiskOfferingVO diskOffering = diskOfferingDao.findById(serviceOffering.getDiskOfferingId()); |
| boolean isLocalStorage = ConfigurationManagerImpl.SystemVMUseLocalStorage.valueIn(dest.getDataCenter().getId()); |
| if (isLocalStorage == diskOffering.isUseLocalStorage()) { |
| logger.debug(String.format("Service offering %s (uuid: %s) will be used on virtual router", serviceOffering.getName(), serviceOffering.getUuid())); |
| serviceOfferingId = serviceOffering.getId(); |
| } |
| } |
| } |
| |
| protected void findServiceOfferingId() { |
| serviceOfferingId = networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()).getServiceOfferingId(); |
| if (serviceOfferingId == null) { |
| findAccountServiceOfferingId(guestNetwork.getAccountId()); |
| } |
| if (serviceOfferingId == null) { |
| findDefaultServiceOfferingId(); |
| } |
| } |
| |
| protected void findVirtualProvider() { |
| // Check if providers are supported in the physical networks |
| final Type type = Type.VirtualRouter; |
| final Long physicalNetworkId = networkModel.getPhysicalNetworkId(guestNetwork); |
| final PhysicalNetworkServiceProvider provider = physicalProviderDao.findByServiceProvider(physicalNetworkId, type.toString()); |
| |
| if (provider == null) { |
| throw new CloudRuntimeException(String.format("Cannot find service provider %s in physical network %s", type.toString(), physicalNetworkId)); |
| } |
| |
| vrProvider = vrProviderDao.findByNspIdAndType(provider.getId(), type); |
| if (vrProvider == null) { |
| throw new CloudRuntimeException(String.format("Cannot find virtual router provider %s as service provider %s", type.toString(), provider.getId())); |
| } |
| } |
| |
| protected void deployAllVirtualRouters() throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException { |
| final int routersToDeploy = getNumberOfRoutersToDeploy(); |
| for (int i = 0; i < routersToDeploy; i++) { |
| // Don't start the router as we are holding the network lock that |
| // needs to be released at the end of router allocation |
| final DomainRouterVO router = nwHelper.deployRouter(this, false); |
| //check if the network update is in progress. |
| //if update is in progress add the update_pending flag to DomainRouterVO. |
| NetworkDetailVO detail = networkDetailsDao.findDetail(guestNetwork.getId(),Network.updatingInSequence); |
| if("true".equalsIgnoreCase(detail!=null ? detail.getValue() : null)) { |
| router.setUpdateState(VirtualRouter.UpdateState.UPDATE_IN_PROGRESS); |
| routerDao.persist(router); |
| } |
| if (router != null) { |
| routerDao.addRouterToGuestNetwork(router, guestNetwork); |
| //Fix according to changes by Sheng Yang in commit ID cb4513379996b262ae378daf00c6388c6b7313cf |
| routers.add(router); |
| } |
| } |
| } |
| |
| /** |
| * Lists all pods given a Data Center Id, a {@link VirtualMachine.Type} and |
| * a list of {@link VirtualMachine.State} |
| * @param id |
| * @param type |
| * @param states |
| * @return |
| */ |
| protected List<HostPodVO> listByDataCenterIdVMTypeAndStates(final long id, final VirtualMachine.Type type, final VirtualMachine.State... states) { |
| final SearchBuilder<VMInstanceVO> vmInstanceSearch = vmDao.createSearchBuilder(); |
| vmInstanceSearch.and("type", vmInstanceSearch.entity().getType(), SearchCriteria.Op.EQ); |
| vmInstanceSearch.and("states", vmInstanceSearch.entity().getState(), SearchCriteria.Op.IN); |
| |
| final SearchBuilder<HostPodVO> podIdSearch = podDao.createSearchBuilder(); |
| podIdSearch.and("dc", podIdSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ); |
| podIdSearch.select(null, SearchCriteria.Func.DISTINCT, podIdSearch.entity().getId()); |
| podIdSearch.join("vmInstanceSearch", vmInstanceSearch, podIdSearch.entity().getId(), vmInstanceSearch.entity().getPodIdToDeployIn(), JoinBuilder.JoinType.INNER); |
| podIdSearch.done(); |
| |
| final SearchCriteria<HostPodVO> sc = podIdSearch.create(); |
| sc.setParameters("dc", id); |
| sc.setJoinParameters("vmInstanceSearch", "type", type); |
| sc.setJoinParameters("vmInstanceSearch", "states", (Object[]) states); |
| return podDao.search(sc, null); |
| } |
| |
| protected void planDeploymentRouters() { |
| if (isBasic()) { |
| routers.addAll(routerDao.listByNetworkAndPodAndRole(guestNetwork.getId(), getPodId(), Role.VIRTUAL_ROUTER)); |
| } else { |
| routers.addAll(routerDao.listByNetworkAndRole(guestNetwork.getId(), Role.VIRTUAL_ROUTER)); |
| } |
| } |
| |
| /** |
| * Routers need reset if at least one of the routers is not redundant or |
| * stopped. |
| */ |
| protected boolean routersNeedReset() { |
| boolean needReset = true; |
| for (final DomainRouterVO router : routers) { |
| if (!router.getIsRedundantRouter() || router.getState() != VirtualMachine.State.Stopped) { |
| needReset = false; |
| break; |
| } |
| } |
| |
| return needReset; |
| } |
| } |
