server: reserve backup, bucket resource limits during operations Changes to check resource limits with reservations for the following resource types: - backup - backup_storage - bnucket - object_storage Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 539503e..088517e 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java
@@ -78,6 +78,7 @@ import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.poll.BackgroundPollManager; import org.apache.cloudstack.poll.BackgroundPollTask; +import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; @@ -85,6 +86,8 @@ import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; import com.amazonaws.util.CollectionUtils; import com.cloud.alert.AlertManager; @@ -119,6 +122,7 @@ import com.cloud.offering.DiskOffering; import com.cloud.offering.ServiceOffering; import com.cloud.projects.Project; +import com.cloud.resourcelimit.CheckedReservation; import com.cloud.serializer.GsonHelper; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; @@ -170,8 +174,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; -import org.apache.commons.lang3.builder.ReflectionToStringBuilder; -import org.apache.commons.lang3.builder.ToStringStyle; public class BackupManagerImpl extends ManagerBase implements BackupManager { @@ -237,6 +239,8 @@ private AlertManager alertManager; @Inject private GuestOSDao _guestOSDao; + @Inject + ReservationDao reservationDao; private AsyncJobDispatcher asyncJobDispatcher; private Timer backupTimer; @@ -792,14 +796,6 @@ Long backupScheduleId = getBackupScheduleId(job); boolean isScheduledBackup = backupScheduleId != null; Account owner = accountManager.getAccount(vm.getAccountId()); - try { - resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup); - } catch (ResourceAllocationException e) { - if (isScheduledBackup) { - sendExceededBackupLimitAlert(owner.getUuid(), Resource.ResourceType.backup); - } - throw e; - } Long backupSize = 0L; for (final Volume volume: volumeDao.findByInstance(vmId)) { @@ -811,42 +807,57 @@ backupSize += volumeSize; } } - try { - resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup_storage, backupSize); - } catch (ResourceAllocationException e) { - if (isScheduledBackup) { - sendExceededBackupLimitAlert(owner.getUuid(), Resource.ResourceType.backup_storage); - } - throw e; - } - - ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), - EventTypes.EVENT_VM_BACKUP_CREATE, "creating backup for VM ID:" + vm.getUuid(), - vmId, ApiCommandResourceType.VirtualMachine.toString(), - true, 0); - - Pair<Boolean, Backup> result = backupProvider.takeBackup(vm, cmd.getQuiesceVM()); - if (!result.first()) { - throw new CloudRuntimeException("Failed to create VM backup"); - } - Backup backup = result.second(); - if (backup != null) { - BackupVO vmBackup = backupDao.findById(result.second().getId()); - vmBackup.setBackupScheduleId(backupScheduleId); - if (cmd.getName() != null) { - vmBackup.setName(cmd.getName()); - } - vmBackup.setDescription(cmd.getDescription()); - backupDao.update(vmBackup.getId(), vmBackup); - resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); - resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); - } + createCheckedBackup(cmd, owner, isScheduledBackup, backupSize, vm, vmId, backupProvider, backupScheduleId); if (isScheduledBackup) { deleteOldestBackupFromScheduleIfRequired(vmId, backupScheduleId); } return true; } + private void createCheckedBackup(CreateBackupCmd cmd, Account owner, boolean isScheduledBackup, Long backupSize, + VMInstanceVO vm, Long vmId, BackupProvider backupProvider, Long backupScheduleId) + throws ResourceAllocationException { + try (CheckedReservation backupReservation = new CheckedReservation(owner, Resource.ResourceType.backup, + 1L, reservationDao, resourceLimitMgr); + CheckedReservation backupStorageReservation = new CheckedReservation(owner, + Resource.ResourceType.backup_storage, backupSize, reservationDao, resourceLimitMgr)) { + + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), + EventTypes.EVENT_VM_BACKUP_CREATE, "creating backup for VM ID:" + vm.getUuid(), + vmId, ApiCommandResourceType.VirtualMachine.toString(), + true, 0); + + Pair<Boolean, Backup> result = backupProvider.takeBackup(vm, cmd.getQuiesceVM()); + if (!result.first()) { + throw new CloudRuntimeException("Failed to create VM backup"); + } + Backup backup = result.second(); + if (backup != null) { + BackupVO vmBackup = backupDao.findById(result.second().getId()); + vmBackup.setBackupScheduleId(backupScheduleId); + if (cmd.getName() != null) { + vmBackup.setName(cmd.getName()); + } + vmBackup.setDescription(cmd.getDescription()); + backupDao.update(vmBackup.getId(), vmBackup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + } + } catch (Exception e) { + if (e instanceof ResourceAllocationException) { + ResourceAllocationException rae = (ResourceAllocationException)e; + if (isScheduledBackup && (Resource.ResourceType.backup.equals(rae.getResourceType()) || + Resource.ResourceType.backup_storage.equals(rae.getResourceType()))) { + sendExceededBackupLimitAlert(owner.getUuid(), rae.getResourceType()); + } + throw rae; + } else if (e instanceof CloudRuntimeException) { + throw (CloudRuntimeException)e; + } + throw new CloudRuntimeException("Failed to create backup for VM with ID: " + vm.getUuid(), e); + } + } + /** * Sends an alert when the backup limit has been exceeded for a given account. * @@ -1538,19 +1549,35 @@ throw new CloudRuntimeException(String.format("Backup offering with ID [%s] does not exist.", backup.getBackupOfferingId())); } final BackupProvider backupProvider = getBackupProvider(backup.getZoneId()); - boolean result = backupProvider.deleteBackup(backup, forced); - if (result) { - resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup); - Long backupSize = backup.getSize() != null ? backup.getSize() : 0L; - resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup_storage, backupSize); - if (backupDao.remove(backup.getId())) { - checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(vm, backup); - return true; - } else { - return false; + return deleteCheckedBackup(forced, backupProvider, backup, vm); + } + + private boolean deleteCheckedBackup(Boolean forced, BackupProvider backupProvider, BackupVO backup, VMInstanceVO vm) { + Account owner = accountManager.getAccount(backup.getAccountId()); + long backupSize = backup.getSize() != null ? backup.getSize() : 0L; + try (CheckedReservation backupReservation = new CheckedReservation(owner, Resource.ResourceType.backup, + backup.getId(), null, -1L, reservationDao, resourceLimitMgr); + CheckedReservation backupStorageReservation = new CheckedReservation(owner, + Resource.ResourceType.backup_storage, backup.getId(), null, -1 * backupSize, + reservationDao, resourceLimitMgr)) { + boolean result = backupProvider.deleteBackup(backup, forced); + if (result) { + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup_storage, backupSize); + if (backupDao.remove(backup.getId())) { + checkAndGenerateUsageForLastBackupDeletedAfterOfferingRemove(vm, backup); + return true; + } else { + return false; + } } + throw new CloudRuntimeException("Failed to delete the backup"); + } catch (Exception e) { + if (e instanceof CloudRuntimeException) { + throw (CloudRuntimeException) e; + } + throw new CloudRuntimeException("Failed to delete the backup due to: " + e.getMessage(), e); } - throw new CloudRuntimeException("Failed to delete the backup"); } /**
diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java index 5a18f16..59adb3a 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java
@@ -16,6 +16,27 @@ // under the License. package org.apache.cloudstack.storage.object; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.reservation.dao.ReservationDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.commons.lang3.ObjectUtils; +import org.jetbrains.annotations.NotNull; + import com.amazonaws.services.s3.internal.BucketNameUtils; import com.amazonaws.services.s3.model.IllegalBucketNameException; import com.cloud.agent.api.to.BucketTO; @@ -24,6 +45,7 @@ import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.CheckedReservation; import com.cloud.resourcelimit.ResourceLimitManagerImpl; import com.cloud.storage.BucketVO; import com.cloud.storage.DataStoreRole; @@ -36,22 +58,6 @@ import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; -import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; -import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; public class BucketApiServiceImpl extends ManagerBase implements BucketApiService, Configurable { @@ -68,6 +74,8 @@ @Inject private BucketStatisticsDao _bucketStatisticsDao; + @Inject + ReservationDao reservationDao; private ScheduledExecutorService _executor = null; @@ -136,13 +144,33 @@ return null; } - resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.bucket); - resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); + long size = ObjectUtils.defaultIfNull(cmd.getQuota(), 0) * Resource.ResourceType.bytesToGiB; + return createCheckedBucket(cmd, owner, size, ownerId); + } - BucketVO bucket = new BucketVO(ownerId, owner.getDomainId(), cmd.getObjectStoragePoolId(), cmd.getBucketName(), cmd.getQuota(), - cmd.isVersioning(), cmd.isEncryption(), cmd.isObjectLocking(), cmd.getPolicy()); - _bucketDao.persist(bucket); - return bucket; + @NotNull + private BucketVO createCheckedBucket(CreateBucketCmd cmd, Account owner, long size, long ownerId) throws ResourceAllocationException { + try (CheckedReservation bucketReservation = new CheckedReservation(owner, Resource.ResourceType.bucket, + 1L, reservationDao, resourceLimitManager); + CheckedReservation objectStorageReservation = new CheckedReservation(owner, + Resource.ResourceType.object_storage, size, reservationDao, resourceLimitManager)) { + BucketVO bucket = new BucketVO(ownerId, owner.getDomainId(), cmd.getObjectStoragePoolId(), cmd.getBucketName(), cmd.getQuota(), + cmd.isVersioning(), cmd.isEncryption(), cmd.isObjectLocking(), cmd.getPolicy()); + _bucketDao.persist(bucket); + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); + if (size > 0) { + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, + (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); + } + return bucket; + } catch (Exception e) { + if (e instanceof ResourceAllocationException) { + throw (ResourceAllocationException)e; + } else if (e instanceof CloudRuntimeException) { + throw (CloudRuntimeException)e; + } + throw new CloudRuntimeException(String.format("Failed to create bucket due to: %s", e.getMessage()), e); + } } @Override @@ -160,7 +188,6 @@ try { bucketTO = new BucketTO(objectStore.createBucket(bucket, objectLock)); bucketCreated = true; - resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); if (cmd.isVersioning()) { objectStore.setBucketVersioning(bucketTO); @@ -172,7 +199,6 @@ if (cmd.getQuota() != null) { objectStore.setQuota(bucketTO, cmd.getQuota()); - resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); if (objectStoreVO.getTotalSize() != null && objectStoreVO.getTotalSize() != 0 && objectStoreVO.getAllocatedSize() != null) { Long allocatedSize = objectStoreVO.getAllocatedSize() / Resource.ResourceType.bytesToGiB; Long totalSize = objectStoreVO.getTotalSize() / Resource.ResourceType.bytesToGiB; @@ -193,11 +219,16 @@ _objectStoreDao.updateAllocatedSize(objectStoreVO, cmd.getQuota() * Resource.ResourceType.bytesToGiB); } } catch (Exception e) { - logger.debug("Failed to create bucket with name: "+bucket.getName(), e); + logger.debug("Failed to create bucket with name: {}", bucket.getName(), e); if(bucketCreated) { objectStore.deleteBucket(bucketTO); } _bucketDao.remove(bucket.getId()); + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); + if (bucket.getQuota() != null) { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, + (bucket.getQuota() * Resource.ResourceType.bytesToGiB)); + } throw new CloudRuntimeException("Failed to create bucket with name: "+bucket.getName()+". "+e.getMessage()); } return bucket; @@ -207,22 +238,39 @@ @ActionEvent(eventType = EventTypes.EVENT_BUCKET_DELETE, eventDescription = "deleting bucket") public boolean deleteBucket(long bucketId, Account caller) { Bucket bucket = _bucketDao.findById(bucketId); - BucketTO bucketTO = new BucketTO(bucket); if (bucket == null) { throw new InvalidParameterValueException("Unable to find bucket with ID: " + bucketId); } _accountMgr.checkAccess(caller, null, true, bucket); ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); - if (objectStore.deleteBucket(bucketTO)) { - resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); - if (bucket.getQuota() != null) { - resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (bucket.getQuota() * Resource.ResourceType.bytesToGiB)); - _objectStoreDao.updateAllocatedSize(objectStoreVO, -(bucket.getQuota() * Resource.ResourceType.bytesToGiB)); + return deleteCheckedBucket(objectStore, bucket, objectStoreVO); + } + + private boolean deleteCheckedBucket(ObjectStoreEntity objectStore, Bucket bucket, ObjectStoreVO objectStoreVO) { + Account owner = _accountMgr.getAccount(bucket.getAccountId()); + try (CheckedReservation bucketReservation = new CheckedReservation(owner, Resource.ResourceType.bucket, + bucket.getId(), null, -1L, reservationDao, resourceLimitManager); + CheckedReservation objectStorageReservation = new CheckedReservation(owner, + Resource.ResourceType.object_storage, bucket.getId(), null, + -1*(ObjectUtils.defaultIfNull(bucket.getQuota(), 0) * Resource.ResourceType.bytesToGiB), reservationDao, resourceLimitManager)) { + BucketTO bucketTO = new BucketTO(bucket); + if (objectStore.deleteBucket(bucketTO)) { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); + if (bucket.getQuota() != null) { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (bucket.getQuota() * Resource.ResourceType.bytesToGiB)); + _objectStoreDao.updateAllocatedSize(objectStoreVO, -(bucket.getQuota() * Resource.ResourceType.bytesToGiB)); + } + _bucketDao.remove(bucket.getId()); + return true; } - return _bucketDao.remove(bucketId); + return false; + } catch (Exception e) { + if (e instanceof CloudRuntimeException) { + throw (CloudRuntimeException) e; + } + throw new CloudRuntimeException(String.format("Failed to delete bucket due to: %s", e.getMessage()), e); } - return false; } @Override
diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 8b13fd4..7395f22 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java
@@ -16,9 +16,72 @@ // under the License. package org.apache.cloudstack.backup; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.TimeZone; +import java.util.UUID; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; +import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; +import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; +import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; +import org.apache.cloudstack.api.command.user.backup.ListBackupScheduleCmd; +import org.apache.cloudstack.api.response.BackupResponse; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupDetailsDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; +import org.apache.cloudstack.reservation.ReservationVO; +import org.apache.cloudstack.reservation.dao.ReservationDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.lang3.StringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.alert.AlertManager; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.vo.UserVmJoinVO; -import com.cloud.alert.AlertManager; import com.cloud.capacity.CapacityVO; import com.cloud.configuration.Resource; import com.cloud.dc.DataCenter; @@ -31,6 +94,7 @@ import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; @@ -42,7 +106,6 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; -import com.cloud.exception.ResourceAllocationException; import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; @@ -61,6 +124,7 @@ import com.cloud.user.dao.AccountDao; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; +import com.cloud.utils.db.DbUtil; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; @@ -68,67 +132,11 @@ import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VmDiskInfo; import com.cloud.vm.VirtualMachineManager; -import com.cloud.vm.dao.VMInstanceDetailsDao; +import com.cloud.vm.VmDiskInfo; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.google.gson.Gson; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; -import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; -import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; -import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; -import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; -import org.apache.cloudstack.api.command.user.backup.ListBackupScheduleCmd; -import org.apache.cloudstack.api.response.BackupResponse; -import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupDetailsDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDao; -import org.apache.cloudstack.backup.dao.BackupScheduleDao; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.mockito.Spy; -import org.springframework.test.util.ReflectionTestUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.TimeZone; -import java.util.UUID; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.atLeastOnce; @RunWith(MockitoJUnitRunner.class) public class BackupManagerTest { @@ -239,6 +247,9 @@ DomainDao domainDao; @Mock + ReservationDao reservationDao; + + @Mock private GuestOSDao _guestOSDao; private Gson gson; @@ -248,6 +259,8 @@ private AutoCloseable closeable; private ConfigDepotImpl configDepotImpl; private boolean updatedConfigKeyDepot = false; + private MockedStatic<DbUtil> dbUtilMockedStatic; + private final List<String> mockedGlobalLocks = new ArrayList<>(); @Before public void setup() throws Exception { @@ -279,6 +292,29 @@ backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider); ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap); + when(reservationDao.persist(any(ReservationVO.class))) + .thenAnswer((Answer<ReservationVO>) invocation -> { + ReservationVO reservationVO = (ReservationVO)invocation.getArguments()[0]; + ReflectionTestUtils.setField(reservationVO, "id", 10L); + return reservationVO; + }); + dbUtilMockedStatic = Mockito.mockStatic(DbUtil.class); + dbUtilMockedStatic.when(() -> DbUtil.getGlobalLock(anyString(), anyInt())).thenAnswer((Answer<Boolean>) invocation -> { + String lockName = invocation.getArgument(0); + if (!StringUtils.isBlank(lockName) && !mockedGlobalLocks.contains(lockName)) { + mockedGlobalLocks.add(lockName); + return true; + } + return false; + }); + dbUtilMockedStatic.when(() -> DbUtil.releaseGlobalLock(anyString())).thenAnswer((Answer<Boolean>) invocation -> { + String lockName = invocation.getArgument(0); + if (!StringUtils.isBlank(lockName)) { + mockedGlobalLocks.remove(lockName); + } + return true; + }); + Account account = mock(Account.class); User user = mock(User.class); CallContext.register(user, account); @@ -286,6 +322,7 @@ @After public void tearDown() throws Exception { + dbUtilMockedStatic.close(); closeable.close(); if (updatedConfigKeyDepot) { ReflectionTestUtils.setField(BackupManager.BackupFrameworkEnabled, "s_depot", configDepotImpl); @@ -622,6 +659,7 @@ Long oldestBackupId = 7L; Long newBackupSize = 1000000000L; Long oldBackupSize = 400000000L; + long domainId = 101L; when(vmInstanceDao.findById(vmId)).thenReturn(vmInstanceVOMock); when(vmInstanceVOMock.getDataCenterId()).thenReturn(zoneId); @@ -636,6 +674,7 @@ Mockito.doReturn(scheduleId).when(backupManager).getBackupScheduleId(asyncJobVOMock); when(accountManager.getAccount(accountId)).thenReturn(accountVOMock); + when(accountVOMock.getDomainId()).thenReturn(domainId); BackupScheduleVO schedule = mock(BackupScheduleVO.class); when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); @@ -677,8 +716,10 @@ assertTrue(backupManager.createBackup(cmd, asyncJobVOMock)); - Mockito.verify(resourceLimitMgr, times(1)).checkResourceLimit(accountVOMock, Resource.ResourceType.backup); - Mockito.verify(resourceLimitMgr, times(1)).checkResourceLimit(accountVOMock, Resource.ResourceType.backup_storage, newBackupSize); + Mockito.verify(resourceLimitMgr, times(1)) + .checkResourceLimitWithTag(accountVOMock, domainId, true, Resource.ResourceType.backup, null, 1L); + Mockito.verify(resourceLimitMgr, times(1)) + .checkResourceLimitWithTag(accountVOMock, domainId, true, Resource.ResourceType.backup_storage, null, newBackupSize); Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); @@ -694,6 +735,7 @@ Long scheduleId = 3L; Long backupOfferingId = 4L; Long accountId = 5L; + long domainId = 101L; when(vmInstanceDao.findById(vmId)).thenReturn(vmInstanceVOMock); when(vmInstanceVOMock.getDataCenterId()).thenReturn(zoneId); @@ -708,7 +750,9 @@ Account account = Mockito.mock(Account.class); when(accountManager.getAccount(accountId)).thenReturn(account); - Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup)).when(resourceLimitMgr).checkResourceLimit(account, Resource.ResourceType.backup); + when(account.getDomainId()).thenReturn(domainId); + Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup)).when(resourceLimitMgr) + .checkResourceLimitWithTag(account, domainId, true, Resource.ResourceType.backup, null, 1L); CreateBackupCmd cmd = Mockito.mock(CreateBackupCmd.class); when(cmd.getVmId()).thenReturn(vmId); @@ -732,6 +776,8 @@ Long scheduleId = 3L; Long backupOfferingId = 4L; Long accountId = 5L; + long domainId = 101L; + long size = 10000L; VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); when(vmInstanceDao.findById(vmId)).thenReturn(vm); @@ -739,6 +785,11 @@ when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); when(vm.getAccountId()).thenReturn(accountId); + VolumeVO volume = mock(VolumeVO.class); + when(volumeDao.findByInstance(vmId)).thenReturn(List.of(volume)); + when(volume.getState()).thenReturn(Volume.State.Ready); + when(volumeApiService.getVolumePhysicalSize(null, null, null)).thenReturn(size); + overrideBackupFrameworkConfigValue(); BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); @@ -749,7 +800,10 @@ Account account = Mockito.mock(Account.class); when(accountManager.getAccount(accountId)).thenReturn(account); - Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup_storage)).when(resourceLimitMgr).checkResourceLimit(account, Resource.ResourceType.backup_storage, 0L); + when(account.getDomainId()).thenReturn(domainId); + Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup_storage)) + .when(resourceLimitMgr) + .checkResourceLimitWithTag(account, domainId, true, Resource.ResourceType.backup_storage, null, size); CreateBackupCmd cmd = Mockito.mock(CreateBackupCmd.class); when(cmd.getVmId()).thenReturn(vmId); @@ -1487,6 +1541,7 @@ when(backup.getVmId()).thenReturn(vmId); when(backup.getZoneId()).thenReturn(zoneId); when(backup.getAccountId()).thenReturn(accountId); + when(accountManager.getAccount(accountId)).thenReturn(accountVOMock); when(backup.getBackupOfferingId()).thenReturn(backupOfferingId); when(backup.getSize()).thenReturn(100L);
diff --git a/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java index b630ddc..6dbdd45 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java
@@ -16,19 +16,36 @@ // under the License. package org.apache.cloudstack.storage.object; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.List; + import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.reservation.ReservationVO; +import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.commons.lang3.StringUtils; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; import org.springframework.test.util.ReflectionTestUtils; import com.cloud.agent.api.to.BucketTO; @@ -40,6 +57,9 @@ import com.cloud.storage.dao.BucketDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.utils.db.DbUtil; @RunWith(MockitoJUnitRunner.class) public class BucketApiServiceImplTest { @@ -62,33 +82,90 @@ @Mock private BucketDao bucketDao; + @Mock + ReservationDao reservationDao; + + @Mock + private AccountVO mockAccountVO; + + private MockedStatic<DbUtil> dbUtilMockedStatic; + private final List<String> mockedGlobalLocks = new ArrayList<>(); + private static final long ACCOUNT_ID = 1001L; + private static final long DOMAIN_ID = 10L; + + @Before + public void setup() { + when(accountManager.getActiveAccountById(ACCOUNT_ID)).thenReturn(mockAccountVO); + when(mockAccountVO.getDomainId()).thenReturn(DOMAIN_ID); + when(reservationDao.persist(any(ReservationVO.class))) + .thenAnswer((Answer<ReservationVO>) invocation -> { + ReservationVO reservationVO = (ReservationVO)invocation.getArguments()[0]; + ReflectionTestUtils.setField(reservationVO, "id", 10L); + return reservationVO; + }); + dbUtilMockedStatic = Mockito.mockStatic(DbUtil.class); + dbUtilMockedStatic.when(() -> DbUtil.getGlobalLock(anyString(), anyInt())) + .thenAnswer((Answer<Boolean>) invocation -> { + String lockName = invocation.getArgument(0); + if (!StringUtils.isBlank(lockName) && !mockedGlobalLocks.contains(lockName)) { + mockedGlobalLocks.add(lockName); + return true; + } + return false; + }); + dbUtilMockedStatic.when(() -> DbUtil.releaseGlobalLock(anyString())) + .thenAnswer((Answer<Boolean>) invocation -> { + String lockName = invocation.getArgument(0); + if (!StringUtils.isBlank(lockName)) { + mockedGlobalLocks.remove(lockName); + } + return true; + }); + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext.register(user, account); + } + + @After + public void tearDown() throws Exception { + dbUtilMockedStatic.close(); + CallContext.unregister(); + } + @Test public void testAllocBucket() throws ResourceAllocationException { String bucketName = "bucket1"; - Long accountId = 1L; Long poolId = 2L; Long objectStoreId = 3L; + int quota = 1; CreateBucketCmd cmd = Mockito.mock(CreateBucketCmd.class); Mockito.when(cmd.getBucketName()).thenReturn(bucketName); - Mockito.when(cmd.getEntityOwnerId()).thenReturn(accountId); + Mockito.when(cmd.getEntityOwnerId()).thenReturn(ACCOUNT_ID); Mockito.when(cmd.getObjectStoragePoolId()).thenReturn(poolId); - Mockito.when(cmd.getQuota()).thenReturn(1); - - Account account = Mockito.mock(Account.class); - Mockito.when(accountManager.getActiveAccountById(accountId)).thenReturn(account); + Mockito.when(cmd.getQuota()).thenReturn(quota); ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); - Mockito.when(objectStore.createUser(accountId)).thenReturn(true); + Mockito.when(objectStore.createUser(ACCOUNT_ID)).thenReturn(true); bucketApiService.allocBucket(cmd); - Mockito.verify(resourceLimitManager, Mockito.times(1)).checkResourceLimit(account, Resource.ResourceType.bucket); - Mockito.verify(resourceLimitManager, Mockito.times(1)).checkResourceLimit(account, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + long size = quota * Resource.ResourceType.bytesToGiB; + Mockito.verify(resourceLimitManager, Mockito.times(1)) + .checkResourceLimitWithTag(mockAccountVO, DOMAIN_ID, true, + Resource.ResourceType.bucket, null, 1L); + Mockito.verify(resourceLimitManager, Mockito.times(1)) + .checkResourceLimitWithTag(mockAccountVO, DOMAIN_ID, true, + Resource.ResourceType.object_storage, null, size); + Mockito.verify(resourceLimitManager, Mockito.times(1)) + .incrementResourceCount(ACCOUNT_ID, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)) + .incrementResourceCount(ACCOUNT_ID, Resource.ResourceType.object_storage, size); } @Test @@ -96,21 +173,21 @@ Long objectStoreId = 1L; Long poolId = 2L; Long bucketId = 3L; - Long accountId = 4L; String bucketName = "bucket1"; + int quota = 3; CreateBucketCmd cmd = Mockito.mock(CreateBucketCmd.class); Mockito.when(cmd.getObjectStoragePoolId()).thenReturn(poolId); Mockito.when(cmd.getEntityId()).thenReturn(bucketId); - Mockito.when(cmd.getQuota()).thenReturn(1); + Mockito.when(cmd.getQuota()).thenReturn(quota); BucketVO bucket = new BucketVO(bucketName); Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); - ReflectionTestUtils.setField(bucket, "accountId", accountId); + ReflectionTestUtils.setField(bucket, "accountId", ACCOUNT_ID); ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); - Mockito.when(objectStoreVO.getTotalSize()).thenReturn(2000000000L); + Mockito.when(objectStoreVO.getTotalSize()).thenReturn(10 * Resource.ResourceType.bytesToGiB); Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); @@ -118,23 +195,23 @@ bucketApiService.createBucket(cmd); - Mockito.verify(resourceLimitManager, Mockito.times(1)).incrementResourceCount(accountId, Resource.ResourceType.bucket); - Mockito.verify(resourceLimitManager, Mockito.times(1)).incrementResourceCount(accountId, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); - Assert.assertEquals(bucket.getState(), Bucket.State.Created); + Assert.assertEquals(Bucket.State.Created, bucket.getState()); } @Test public void testDeleteBucket() { Long bucketId = 1L; - Long accountId = 2L; Long objectStoreId = 3L; String bucketName = "bucket1"; + int quota = 2; - BucketVO bucket = new BucketVO(bucketName); + BucketVO bucket = mock(BucketVO.class); + when(bucket.getName()).thenReturn(bucketName); + when(bucket.getObjectStoreId()).thenReturn(objectStoreId); + when(bucket.getQuota()).thenReturn(quota); + when(bucket.getAccountId()).thenReturn(ACCOUNT_ID); + when(accountManager.getAccount(ACCOUNT_ID)).thenReturn(mock(AccountVO.class)); Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); - ReflectionTestUtils.setField(bucket, "objectStoreId", objectStoreId); - ReflectionTestUtils.setField(bucket, "quota", 1); - ReflectionTestUtils.setField(bucket, "accountId", accountId); ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); @@ -145,15 +222,17 @@ bucketApiService.deleteBucket(bucketId, null); - Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.bucket); - Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + Mockito.verify(resourceLimitManager, Mockito.times(1)) + .decrementResourceCount(ACCOUNT_ID, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)) + .decrementResourceCount(ACCOUNT_ID, Resource.ResourceType.object_storage, + quota * Resource.ResourceType.bytesToGiB); } @Test public void testUpdateBucket() throws ResourceAllocationException { Long bucketId = 1L; Long objectStoreId = 2L; - Long accountId = 3L; Integer bucketQuota = 2; Integer cmdQuota = 1; String bucketName = "bucket1"; @@ -164,12 +243,10 @@ BucketVO bucket = new BucketVO(bucketName); ReflectionTestUtils.setField(bucket, "quota", bucketQuota); - ReflectionTestUtils.setField(bucket, "accountId", accountId); + ReflectionTestUtils.setField(bucket, "accountId", ACCOUNT_ID); ReflectionTestUtils.setField(bucket, "objectStoreId", objectStoreId); Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); - Account account = Mockito.mock(Account.class); - ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); Mockito.when(objectStoreDao.findById(objectStoreId)).thenReturn(objectStoreVO); @@ -178,6 +255,8 @@ bucketApiService.updateBucket(cmd, null); - Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.object_storage, (bucketQuota - cmdQuota) * Resource.ResourceType.bytesToGiB); + Mockito.verify(resourceLimitManager, Mockito.times(1)) + .decrementResourceCount(ACCOUNT_ID, Resource.ResourceType.object_storage, + (bucketQuota - cmdQuota) * Resource.ResourceType.bytesToGiB); } }