NSX: Cleanup NSX resources during k8s cluster cleanup
diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
index e445e50..e160227 100644
--- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
+++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
@@ -22,4 +22,5 @@
public interface KubernetesClusterHelper extends Adapter {
ControlledEntity findByUuid(String uuid);
+ ControlledEntity findByVmId(long vmId);
}
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
index 0ef916a..60bd81c 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
@@ -17,6 +17,7 @@
package com.cloud.kubernetes.cluster;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
+import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.utils.component.AdapterBase;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -24,12 +25,15 @@
import org.springframework.stereotype.Component;
import javax.inject.Inject;
+import java.util.Objects;
@Component
public class KubernetesClusterHelperImpl extends AdapterBase implements KubernetesClusterHelper, Configurable {
@Inject
private KubernetesClusterDao kubernetesClusterDao;
+ @Inject
+ private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@Override
public ControlledEntity findByUuid(String uuid) {
@@ -37,6 +41,15 @@
}
@Override
+ public ControlledEntity findByVmId(long vmId) {
+ KubernetesClusterVmMapVO clusterVmMapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(vmId);
+ if (Objects.isNull(clusterVmMapVO)) {
+ return null;
+ }
+ return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId());
+ }
+
+ @Override
public String getConfigComponentName() {
return KubernetesClusterHelper.class.getSimpleName();
}
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
index 688a611..45c0b79 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
@@ -23,6 +23,8 @@
public interface KubernetesClusterVmMapDao extends GenericDao<KubernetesClusterVmMapVO, Long> {
public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId);
+
+ public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId);
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
int removeByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
index b9f2ec9..0d90a4c 100644
--- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
+++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
@@ -31,12 +31,17 @@
public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClusterVmMapVO, Long> implements KubernetesClusterVmMapDao {
private final SearchBuilder<KubernetesClusterVmMapVO> clusterIdSearch;
+ private final SearchBuilder<KubernetesClusterVmMapVO> vmIdSearch;
public KubernetesClusterVmMapDaoImpl() {
clusterIdSearch = createSearchBuilder();
clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
clusterIdSearch.and("vmIdsIN", clusterIdSearch.entity().getVmId(), SearchCriteria.Op.IN);
clusterIdSearch.done();
+
+ vmIdSearch = createSearchBuilder();
+ vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ);
+ vmIdSearch.done();
}
@Override
@@ -48,6 +53,13 @@
}
@Override
+ public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId) {
+ SearchCriteria<KubernetesClusterVmMapVO> sc = vmIdSearch.create();
+ sc.setParameters("vmId", vmId);
+ return findOneBy(sc);
+ }
+
+ @Override
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds) {
SearchCriteria<KubernetesClusterVmMapVO> sc = clusterIdSearch.create();
sc.setParameters("clusterId", clusterId);
diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
index 0986c36..571e684 100644
--- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
+++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
@@ -505,10 +505,12 @@
if (!canHandle(network, Network.Service.PortForwarding)) {
return false;
}
+ boolean result = true;
for (PortForwardingRule rule : rules) {
IPAddressVO publicIp = ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId());
UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId());
- if (vm == null || networkModel.getNicInNetwork(vm.getId(), network.getId()) == null) {
+ if ((vm == null && (rule.getState() != FirewallRule.State.Revoke)) ||
+ (vm != null && networkModel.getNicInNetwork(vm.getId(), network.getId()) == null)) {
continue;
}
NsxOpObject nsxObject = getNsxOpObject(network);
@@ -523,8 +525,8 @@
.setNetworkResourceId(nsxObject.getNetworkResourceId())
.setNetworkResourceName(nsxObject.getNetworkResourceName())
.setVpcResource(nsxObject.isVpcResource())
- .setVmId(vm.getId())
- .setVmIp(vm.getPrivateIpAddress())
+ .setVmId(Objects.nonNull(vm) ? vm.getId() : 0)
+ .setVmIp(Objects.nonNull(vm) ? vm.getPrivateIpAddress() : null)
.setPublicIp(publicIp.getAddress().addr())
.setPrivatePort(privatePort)
.setPublicPort(publicPort)
@@ -532,12 +534,12 @@
.setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT))
.build();
if (rule.getState() == FirewallRule.State.Add) {
- return nsxService.createPortForwardRule(networkRule);
+ result &= nsxService.createPortForwardRule(networkRule);
} else if (rule.getState() == FirewallRule.State.Revoke) {
- return nsxService.deletePortForwardRule(networkRule);
+ result &= nsxService.deletePortForwardRule(networkRule);
}
}
- return true;
+ return result;
}
public Pair<VpcVO, NetworkVO> getVpcOrNetwork(Long vpcId, long networkId) {
@@ -613,6 +615,7 @@
@Override
public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) throws ResourceUnavailableException {
+ boolean result = true;
for (LoadBalancingRule loadBalancingRule : rules) {
if (loadBalancingRule.getState() == FirewallRule.State.Active) {
continue;
@@ -638,12 +641,12 @@
.setAlgorithm(loadBalancingRule.getAlgorithm())
.build();
if (loadBalancingRule.getState() == FirewallRule.State.Add) {
- return nsxService.createLbRule(networkRule);
+ result &= nsxService.createLbRule(networkRule);
} else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) {
- return nsxService.deleteLbRule(networkRule);
+ result &= nsxService.deleteLbRule(networkRule);
}
}
- return true;
+ return result;
}
@Override
diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
index 2927db6..844e038 100644
--- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
+++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
@@ -51,6 +51,9 @@
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
+import com.cloud.kubernetes.cluster.KubernetesClusterHelper;
+import com.cloud.network.dao.NsxProviderDao;
+import com.cloud.network.element.NsxProviderVO;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@@ -589,6 +592,8 @@
@Inject
VMScheduleManager vmScheduleManager;
+ @Inject
+ NsxProviderDao nsxProviderDao;
private ScheduledExecutorService _executor = null;
private ScheduledExecutorService _vmIpFetchExecutor = null;
@@ -597,6 +602,7 @@
private boolean _dailyOrHourly = false;
private int capacityReleaseInterval;
private ExecutorService _vmIpFetchThreadExecutor;
+ private List<KubernetesClusterHelper> kubernetesClusterHelpers;
private String _instance;
@@ -610,6 +616,14 @@
private static final int NUM_OF_2K_BLOCKS = 512;
private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES;
+ public List<KubernetesClusterHelper> getKubernetesClusterHelpers() {
+ return kubernetesClusterHelpers;
+ }
+
+ public void setKubernetesClusterHelpers(final List<KubernetesClusterHelper> kubernetesClusterHelpers) {
+ this.kubernetesClusterHelpers = kubernetesClusterHelpers;
+ }
+
@Inject
private OrchestrationService _orchSrvc;
@@ -2528,11 +2542,15 @@
}
// cleanup port forwarding rules
- if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
- s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
- } else {
- success = false;
- s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
+ VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
+ NsxProviderVO nsx = nsxProviderDao.findByZoneId(vmInstanceVO.getDataCenterId());
+ if (Objects.isNull(nsx) || Objects.isNull(kubernetesClusterHelpers.get(0).findByVmId(vmId))) {
+ if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
+ s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
+ } else {
+ success = false;
+ s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
+ }
}
// cleanup load balancer rules
diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
index 7227264..9656d54 100644
--- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
+++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
@@ -106,8 +106,9 @@
<bean id="configurationServerImpl" class="com.cloud.server.ConfigurationServerImpl" />
-
- <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl" />
+ <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl">
+ <property name="kubernetesClusterHelpers" value="#{kubernetesClusterHelperRegistry.registered}" />
+ </bean>
<bean id="consoleProxyManagerImpl" class="com.cloud.consoleproxy.ConsoleProxyManagerImpl">
<property name="consoleProxyAllocators"
