commit 9bed11d1f1195dd5442241f0698f1ef539fa7de9
author Sateesh Chodapuneedi <sateesh.chodapuneedi@accelerite.com> Tue Dec 27 10:03:27 2016 +0530
committer Sateesh Chodapuneedi <sateesh.chodapuneedi@accelerite.com> Thu Mar 02 02:48:06 2017 +0530
tree c29c5451b56143cc7bc68a9a17846dea8ffbf423
parent ee83b02c7640ec69e1ec0a16a071ac2d5b7575b6

CLOUDSTACK-3223 Exception observed while creating CPVM in VMware Setup with DVS

Issue
=====
Exception observed while creating CPVM in VMware Setup with DVS.
Observed error "StartCommand failed due to Exception: com.vmware.vim25.AlreadyExists."
This is due to concurrent attempts to create same dv portgroup on same dvSwitch by
manager threads of CPVM and SSVM when both are started at same time.

Fix
===
Synchronize api calls to create/update dvportgroup.
Also maintaing local cache to avoid multiple fetch api calls to vCenter
when multiple threads try to create same object.

Signed-off-by: Sateesh Chodapuneedi <sateesh.chodapuneedi@accelerite.com>

vmware-base/src/com/cloud/hypervisor/vmware/mo/DistributedVirtualSwitchMO.java

diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/DistributedVirtualSwitchMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/DistributedVirtualSwitchMO.java
index 9e19d65..fbb4265 100644
--- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/DistributedVirtualSwitchMO.java
+++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/DistributedVirtualSwitchMO.java
@@ -20,6 +20,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.log4j.Logger;
 
@@ -36,24 +37,67 @@
 public class DistributedVirtualSwitchMO extends BaseMO {
     @SuppressWarnings("unused")
     private static final Logger s_logger = Logger.getLogger(DistributedVirtualSwitchMO.class);
+    private static ConcurrentHashMap<String, List<String>> s_dvPortGroupCacheMap = null;
 
     public DistributedVirtualSwitchMO(VmwareContext context, ManagedObjectReference morDvs) {
         super(context, morDvs);
+        s_dvPortGroupCacheMap = new ConcurrentHashMap<String, List<String>>();
     }
 
     public DistributedVirtualSwitchMO(VmwareContext context, String morType, String morValue) {
         super(context, morType, morValue);
+        s_dvPortGroupCacheMap = new ConcurrentHashMap<String, List<String>>();
     }
 
     public void createDVPortGroup(DVPortgroupConfigSpec dvPortGroupSpec) throws Exception {
         List<DVPortgroupConfigSpec> dvPortGroupSpecArray = new ArrayList<DVPortgroupConfigSpec>();
         dvPortGroupSpecArray.add(dvPortGroupSpec);
-        _context.getService().addDVPortgroupTask(_mor, dvPortGroupSpecArray);
+        boolean dvPortGroupExists = false;
+        String dvSwitchInstance = _mor.getValue();
+        String dvPortGroupName = dvPortGroupSpec.getName();
+        String uniquedvPortGroupPerDvs = dvSwitchInstance + dvPortGroupName;
+        List<String> dvPortGroupList = null;
+        synchronized (uniquedvPortGroupPerDvs.intern()) {
+            // Looking up local cache rather than firing another API call to see if dvPortGroup exists already.
+            if (s_dvPortGroupCacheMap.containsKey(dvSwitchInstance)) {
+                dvPortGroupList = s_dvPortGroupCacheMap.get(dvSwitchInstance);
+                if (dvPortGroupList != null && dvPortGroupList.contains(dvPortGroupName)) {
+                    dvPortGroupExists = true;
+                }
+            }
+            if (!dvPortGroupExists) {
+                ManagedObjectReference task = _context.getService().addDVPortgroupTask(_mor, dvPortGroupSpecArray);
+                if (!_context.getVimClient().waitForTask(task)) {
+                    throw new Exception("Failed to create dvPortGroup " + dvPortGroupSpec.getName());
+                } else {
+                    if (s_dvPortGroupCacheMap.containsKey(dvSwitchInstance)) {
+                        dvPortGroupList = s_dvPortGroupCacheMap.get(dvSwitchInstance);
+                        if (dvPortGroupList == null) {
+                            dvPortGroupList = new ArrayList<String>();
+                        }
+                        dvPortGroupList.add(dvPortGroupName); //does this update map?
+                    } else {
+                        dvPortGroupList = new ArrayList<String>();
+                        dvPortGroupList.add(dvPortGroupName);
+                        s_dvPortGroupCacheMap.put(dvSwitchInstance, dvPortGroupList);
+                    }
+                }
+                if (s_logger.isTraceEnabled()) {
+                    s_logger.trace("Created dvPortGroup. dvPortGroup cache is :" + s_dvPortGroupCacheMap);
+                }
+            } else if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Detected dvPortGroup [" + dvPortGroupName + "] already present. Not attempting to create again.");
+            }
+        }
     }
 
     public void updateDvPortGroup(ManagedObjectReference dvPortGroupMor, DVPortgroupConfigSpec dvPortGroupSpec) throws Exception {
-        // TODO(sateesh): Update numPorts
-        _context.getService().reconfigureDVPortgroupTask(dvPortGroupMor, dvPortGroupSpec);
+        synchronized (dvPortGroupMor.getValue().intern()) {
+            ManagedObjectReference task = _context.getService().reconfigureDVPortgroupTask(dvPortGroupMor, dvPortGroupSpec);
+            if (!_context.getVimClient().waitForTask(task)) {
+                throw new Exception("Failed to update dvPortGroup " + dvPortGroupMor.getValue());
+            }
+        }
     }
 
     public void updateVMWareDVSwitch(ManagedObjectReference dvSwitchMor, VMwareDVSConfigSpec dvsSpec) throws Exception {