Merge pull request #694 from apache/DLAB-1732
[DLAB-1732]: [Azure] Billing on local endpoint fixed
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index f732871..8ab5f9e 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -283,7 +283,7 @@
### Matplotlib version
matplotlib_version = 2.0.2
### JupyterLab image
-jupyterlab_image = odahu\/base-notebook:1.1.0-rc8
+jupyterlab_image = odahu\\/base-notebook:1.1.0-rc8
### Superset version
superset_version = 0.35.1
### GCS-connector version
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
index 2ca635d..c70e9a9 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/common_lib.py
@@ -30,30 +30,44 @@
def manage_pkg(command, environment, requisites):
try:
- allow = False
- counter = 0
- while not allow:
- if counter > 60:
+ attempt = 0
+ installed = False
+ while not installed:
+ print('Pkg installation attempt: {}'.format(attempt))
+ if attempt > 60:
print("Notebook is broken please recreate it.")
sys.exit(1)
else:
- print('Package manager is:')
- if environment == 'remote':
- if sudo('pgrep "^apt" -a && echo "busy" || echo "ready"') == 'busy':
- counter += 1
- time.sleep(10)
- else:
- allow = True
- sudo('apt-get {0} {1}'.format(command, requisites))
- elif environment == 'local':
- if local('sudo pgrep "^apt" -a && echo "busy" || echo "ready"', capture=True) == 'busy':
- counter += 1
- time.sleep(10)
- else:
- allow = True
- local('sudo apt-get {0} {1}'.format(command, requisites), capture=True)
- else:
- print('Wrong environment')
+ try:
+ allow = False
+ counter = 0
+ while not allow:
+ if counter > 60:
+ print("Notebook is broken please recreate it.")
+ sys.exit(1)
+ else:
+ print('Package manager is:')
+ if environment == 'remote':
+ if sudo('pgrep "^apt" -a && echo "busy" || echo "ready"') == 'busy':
+ counter += 1
+ time.sleep(10)
+ else:
+ allow = True
+ sudo('apt-get {0} {1}'.format(command, requisites))
+ elif environment == 'local':
+ if local('sudo pgrep "^apt" -a && echo "busy" || echo "ready"', capture=True) == 'busy':
+ counter += 1
+ time.sleep(10)
+ else:
+ allow = True
+ local('sudo apt-get {0} {1}'.format(command, requisites), capture=True)
+ else:
+ print('Wrong environment')
+ installed = True
+ except:
+ print("Will try to install with nex attempt.")
+ sudo('dpkg --configure -a')
+ attempt += 1
except:
sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
index f828957..f4cda59 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/ssn_lib.py
@@ -196,16 +196,16 @@
sudo('mv /tmp/ssn.yml ' + os.environ['ssn_dlab_path'] + 'conf/')
put('/root/templates/proxy_location_webapp_template.conf', '/tmp/proxy_location_webapp_template.conf')
sudo('mv /tmp/proxy_location_webapp_template.conf ' + os.environ['ssn_dlab_path'] + 'tmp/')
- if cloud_provider == 'gcp' or 'azure':
- conf_parameter_name = '--spring.config.location='
+ if cloud_provider == 'aws':
+ conf_parameter_name = '--spring.config.location={0}billing_app.yml --conf '.format(dlab_conf_dir)
with open('/root/templates/supervisor_svc.conf', 'r') as f:
text = f.read()
text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
.replace('CONF_PARAMETER_NAME', conf_parameter_name)
with open('/root/templates/supervisor_svc.conf', 'w') as f:
f.write(text)
- elif cloud_provider == 'aws':
- conf_parameter_name = '--spring.config.location={0}billing_app.yml --conf '.format(dlab_conf_dir)
+ elif cloud_provider == 'gcp' or cloud_provider == 'azure':
+ conf_parameter_name = '--spring.config.location='
with open('/root/templates/supervisor_svc.conf', 'r') as f:
text = f.read()
text = text.replace('WEB_CONF', dlab_conf_dir).replace('OS_USR', os_user)\
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index e5fc30f..cd15d42 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -41,6 +41,7 @@
sudo('echo PATH=$PATH:/usr/local/bin/:/opt/spark/bin/ >> /etc/profile')
sudo('echo export PATH >> /etc/profile')
sudo('pip install -UI pip=={} --no-cache-dir'.format(os.environ['conf_pip_version']))
+ sudo('pip install --upgrade setuptools')
sudo('pip install -U {} --no-cache-dir'.format(requisites))
sudo('touch /home/{}/.ensure_dir/pip_path_added'.format(os.environ['conf_os_user']))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
index ae50b95..db8f7cd 100644
--- a/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/azure/ssn_configure.py
@@ -429,10 +429,10 @@
format(ssn_conf['instnace_ip'], ssn_conf['ssh_key_path'], os.environ['ssn_dlab_path'],
ssn_conf['dlab_ssh_user'], os.environ['conf_os_family'], os.environ['request_id'],
os.environ['conf_resource'], ssn_conf['service_base_name'], os.environ['conf_cloud_provider'],
- billing_enabled, azure_auth_path, os.environ['azure_offer_number'],
+ ssn_conf['billing_enabled'], ssn_conf['azure_auth_path'], os.environ['azure_offer_number'],
os.environ['azure_currency'], os.environ['azure_locale'], os.environ['azure_region_info'],
- ldap_login, tenant_id, datalake_application_id, datalake_store_name, json.dumps(cloud_params),
- subscription_id, os.environ['azure_validate_permission_scope'], ssn_conf['default_endpoint_name'],
+ ssn_conf['ldap_login'], ssn_conf['tenant_id'], ssn_conf['datalake_application_id'], ssn_conf['datalake_store_name'], json.dumps(cloud_params),
+ ssn_conf['subscription_id'], os.environ['azure_validate_permission_scope'], ssn_conf['default_endpoint_name'],
os.environ['keycloak_client_name'], os.environ['keycloak_client_secret'],
os.environ['keycloak_auth_server_url'])
local("~/scripts/{}.py {}".format('configure_ui', params))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
index 47b6cde..f9822a0 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/project_prepare.py
@@ -398,7 +398,8 @@
project_conf['tag_name']: project_conf['shared_bucket_name'],
"endpoint_tag": project_conf['endpoint_tag'],
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
- "sbn": project_conf['service_base_name']}
+ "sbn": project_conf['service_base_name'],
+ "name": project_conf['shared_bucket_name']}
params = "--bucket_name {} --tags '{}'".format(project_conf['shared_bucket_name'],
json.dumps(project_conf['shared_bucket_tags']))
try:
@@ -412,7 +413,8 @@
"endpoint_tag": project_conf['endpoint_tag'],
os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
"sbn": project_conf['service_base_name'],
- "project_tag": project_conf['project_tag']}
+ "project_tag": project_conf['project_tag'],
+ "name": project_conf['bucket_name']}
params = "--bucket_name {} --tags '{}'".format(project_conf['bucket_name'],
json.dumps(project_conf['bucket_tags']))
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index ce37274..dd622d2 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -426,8 +426,8 @@
"--request_id {} --billing_dataset_name {} \
--resource {} --service_base_name {} --cloud_provider {} --default_endpoint_name {} " \
"--cloud_params '{}' --keycloak_client_id {} --keycloak_client_secret {} --keycloak_auth_server_url {}". \
- format(instance_hostname, ssn_conf['ssh_key_path'], os.environ['ssn_dlab_path'], ssn_conf['dlab_ssh_user'],
- os.environ['conf_os_family'], billing_enabled, os.environ['request_id'],
+ format(ssn_conf['instance_hostname'], ssn_conf['ssh_key_path'], os.environ['ssn_dlab_path'], ssn_conf['dlab_ssh_user'],
+ os.environ['conf_os_family'], ssn_conf['billing_enabled'], os.environ['request_id'],
os.environ['billing_dataset_name'], os.environ['conf_resource'],
ssn_conf['service_base_name'], os.environ['conf_cloud_provider'], ssn_conf['default_endpoint_name'],
json.dumps(cloud_params), os.environ['keycloak_client_name'], os.environ['keycloak_client_secret'],
diff --git a/infrastructure-provisioning/src/ssn/templates/ssn.yml b/infrastructure-provisioning/src/ssn/templates/ssn.yml
index fffa7d2..7b18d26 100644
--- a/infrastructure-provisioning/src/ssn/templates/ssn.yml
+++ b/infrastructure-provisioning/src/ssn/templates/ssn.yml
@@ -62,5 +62,10 @@
timeout: 3s
connectionTimeout: 3s
+billingService:
+ jerseyClient:
+ timeout: 4m
+ connectionTimeout: 3s
+
# Log out user on inactivity
inactiveUserTimeoutMillSec: 7200000
diff --git a/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml b/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml
index b3b5150..41add93 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml
+++ b/infrastructure-provisioning/terraform/bin/deploy/billing_aws.yml
@@ -32,11 +32,6 @@
password: MONGO_PASSWORD
database: dlabdb
-scheduler:
-# Schedule is comma separated values of time in format hh[:mm[:ss]]. hh - in the 24-hour clock, at 8:15PM is 20:15.
- schedule: 0:00, 1:00, 2:00, 3:00, 4:00, 5:00, 6:00, 7:00, 8:00, 9:00, 10:00, 11:00, 12:00, 13:00, 14:00, 15:00, 16:00,
- 17:00, 18:00, 19:00, 20:00, 21:00, 22:00, 23:00
-
# Adapter for reading source data. Known types: file, s3file
adapterIn:
- type: s3file
diff --git a/infrastructure-provisioning/terraform/bin/deploy/provisioning.yml b/infrastructure-provisioning/terraform/bin/deploy/provisioning.yml
index 9d65fa2..abbbadf 100644
--- a/infrastructure-provisioning/terraform/bin/deploy/provisioning.yml
+++ b/infrastructure-provisioning/terraform/bin/deploy/provisioning.yml
@@ -67,6 +67,11 @@
timeout: 3s
connectionTimeout: 3s
+billingService:
+ jerseyClient:
+ timeout: 4m
+ connectionTimeout: 3s
+
# Log out user on inactivity
inactiveUserTimeoutMillSec: 7200000
@@ -76,7 +81,7 @@
responseDirectory: /opt/dlab/tmp
handlerDirectory: /opt/dlab/handlers
dockerLogDirectory: ${LOG_ROOT_DIR}
-warmupPollTimeout: 25s
+warmupPollTimeout: 2m
resourceStatusPollTimeout: 300m
keyLoaderPollTimeout: 30m
requestEnvStatusTimeout: 50s
diff --git a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml
index fd5fc9b..6edb057 100644
--- a/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml
+++ b/infrastructure-provisioning/terraform/gcp/endpoint/provisioning.yml
@@ -76,7 +76,7 @@
responseDirectory: /opt/dlab/tmp
handlerDirectory: /opt/dlab/handlers
dockerLogDirectory: ${LOG_ROOT_DIR}
-warmupPollTimeout: 25s
+warmupPollTimeout: 2m
resourceStatusPollTimeout: 300m
keyLoaderPollTimeout: 30m
requestEnvStatusTimeout: 50s
diff --git a/services/billing-aws/billing.yml b/services/billing-aws/billing.yml
index 9175c98..3b1943f 100644
--- a/services/billing-aws/billing.yml
+++ b/services/billing-aws/billing.yml
@@ -32,11 +32,6 @@
password: MONGO_PASSWORD
database: dlabdb
-scheduler:
-# Schedule is comma separated values of time in format hh[:mm[:ss]]. hh - in the 24-hour clock, at 8:15PM is 20:15.
- schedule: 0:00, 1:00, 2:00, 3:00, 4:00, 5:00, 6:00, 7:00, 8:00, 9:00, 10:00, 11:00, 12:00, 13:00, 14:00, 15:00, 16:00,
- 17:00, 18:00, 19:00, 20:00, 21:00, 22:00, 23:00
-
# Adapter for reading source data. Known types: file, s3file
adapterIn:
- type: s3file
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java
index cede034..c878370 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingAwsApplication.java
@@ -32,6 +32,6 @@
public static void main(String[] args) throws InitializationException {
SpringApplication.run(BillingAwsApplication.class, args);
- BillingScheduler.startScheduler(args);
+ BillingServiceImpl.startApplication(args);
}
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java
deleted file mode 100644
index 7a7d5ff..0000000
--- a/services/billing-aws/src/main/java/com/epam/dlab/BillingScheduler.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab;
-
-import com.epam.dlab.configuration.BillingToolConfiguration;
-import com.epam.dlab.configuration.BillingToolConfigurationFactory;
-import com.epam.dlab.configuration.SchedulerConfiguration;
-import com.epam.dlab.core.parser.ParserBase;
-import com.epam.dlab.exceptions.AdapterException;
-import com.epam.dlab.exceptions.DlabException;
-import com.epam.dlab.exceptions.InitializationException;
-import com.epam.dlab.exceptions.ParseException;
-import com.epam.dlab.util.ServiceUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-
-/**
- * Billing scheduler for loading billing report.
- */
-public class BillingScheduler implements Runnable {
- private static final Logger LOGGER = LoggerFactory.getLogger(BillingScheduler.class);
-
- /**
- * Timeout for check the schedule in milliseconds.
- */
- private static final long CHECK_TIMEOUT_MILLIS = 60000;
-
- /**
- * Billing scheduler instance.
- */
- private static BillingScheduler scheduler;
- private final boolean enabled;
- private final BillingToolConfiguration configuration;
-
- /**
- * Starts the scheduler for given configuration.
- *
- * @param filename the name of file for billing configuration.
- * @throws InitializationException
- */
- public static void start(String filename) throws InitializationException {
- if (scheduler == null) {
- scheduler = new BillingScheduler(filename);
- scheduler.thread.start();
- } else {
- LOGGER.debug("Billing scheduler already started");
- }
- }
-
- /**
- * Stops the scheduler.
- */
- public static void stop() {
- if (scheduler.thread != null) {
- LOGGER.debug("Billing scheduler will be stopped ...");
- synchronized (scheduler.thread) {
- scheduler.thread.interrupt();
- scheduler.thread = null;
- }
- LOGGER.info("Scheduler has been stopped");
- }
- }
-
-
- /**
- * Thread of the scheduler.
- */
- private Thread thread = new Thread(this, this.getClass().getSimpleName());
-
- /**
- * Name of configuration file.
- */
- private final String confFilename;
-
- /**
- * Current schedule.
- */
- private SchedulerConfiguration schedule;
-
- /**
- * Instantiate billing scheduler for given configuration.
- *
- * @param filename the name of file for billing configuration.
- * @throws InitializationException
- */
- public BillingScheduler(String filename) throws InitializationException {
- this.confFilename = filename;
- LOGGER.debug("Billing report configuration file: {}", filename);
- configuration = BillingToolConfigurationFactory.build(confFilename, BillingToolConfiguration.class);
- this.enabled = configuration.isBillingEnabled();
- setSchedule(configuration);
- }
-
- /**
- * Loads the billing report.
- *
- * @throws InitializationException
- * @throws AdapterException
- * @throws ParseException
- */
- private void load() throws InitializationException, AdapterException, ParseException {
- ParserBase parser = configuration.build();
- long time = schedule.getNearTime().getTimeInMillis();
- if (setSchedule(configuration)) {
- if (time != schedule.getNearTime().getTimeInMillis()) {
- LOGGER.info("Previous billing schedule has been canceled");
- return;
- }
- }
-
- LOGGER.info("Try to laod billing report for configuration: {}", configuration);
- parser.parse();
- if (!parser.getStatistics().isEmpty()) {
- LOGGER.info("Billing report parser statistics:");
- for (int i = 0; i < parser.getStatistics().size(); i++) {
- LOGGER.info(" {}", parser.getStatistics().get(i).toString());
- }
- }
- }
-
- /**
- * Read the schedule from configuration.
- *
- * @param configuration the billing configuration.
- * @return <b>true>/b> if new schedule was loaded, otherwise <b>false</b>.
- * @throws InitializationException
- */
- private boolean setSchedule(BillingToolConfiguration configuration) throws InitializationException {
- SchedulerConfiguration schedulerConfiguration = configuration.getScheduler();
- boolean isModified = false;
- if (schedulerConfiguration == null) {
- throw new InitializationException(String.format("Schedule of billing report in configuration file \"%s " +
- "not found", confFilename));
- }
- if (this.schedule == null) {
- isModified = true;
- LOGGER.debug("Billing report schedule: {}", schedulerConfiguration);
- } else {
- this.schedule.adjustStartTime();
- if (!schedulerConfiguration.equals(this.schedule)) {
- isModified = true;
- LOGGER.debug("New billing report schedule has been loaded: {}", schedulerConfiguration);
- }
- }
-
- try {
- this.schedule = new SchedulerConfiguration();
- this.schedule.setSchedule(schedulerConfiguration.getSchedule());
- this.schedule.build();
- } catch (Exception e) {
- throw new InitializationException("Cannot configure billing scheduler. " + e.getLocalizedMessage(), e);
- }
-
- return isModified;
- }
-
- @Override
- public void run() {
- if (enabled) {
- LOGGER.info("Billing scheduler has been started");
- long startTimeMillis = schedule.getNextTime().getTimeInMillis();
- long timeMillis;
- LOGGER.info("Billing report will be loaded at {}", schedule.getNextTime().getTime());
-
- try {
- while (!Thread.currentThread().isInterrupted()) {
- if (startTimeMillis <= System.currentTimeMillis()) {
- try {
- LOGGER.debug("Try to load billing report for schedule {}",
- schedule.getNextTime().getTime());
- load();
- } catch (InitializationException | AdapterException | ParseException e) {
- LOGGER.error("Error loading billing report: {}", e.getLocalizedMessage(), e);
- }
- startTimeMillis = schedule.getNextTime().getTimeInMillis();
- LOGGER.info("Billing report will be loaded at {}", schedule.getNextTime().getTime());
- } else {
- schedule.adjustStartTime();
- timeMillis = schedule.getNextTime().getTimeInMillis();
- if (startTimeMillis != timeMillis) {
- LOGGER.info("Billing report will be loaded at {}", schedule.getNextTime().getTime());
- startTimeMillis = timeMillis;
- }
- }
-
- try {
- timeMillis = startTimeMillis - System.currentTimeMillis();
- if (timeMillis > 0) {
- timeMillis = Math.min(CHECK_TIMEOUT_MILLIS, timeMillis);
- Thread.sleep(timeMillis);
- }
- } catch (InterruptedException e) {
- LOGGER.warn("Billing scheduler interrupted", e);
- Thread.currentThread().interrupt();
- }
- }
- } catch (Exception e) {
- LOGGER.error("Unhandled billing report error: {}", e.getLocalizedMessage(), e);
- }
- LOGGER.info("Scheduler has been stopped");
- } else {
- LOGGER.info("Billing scheduler is disabled");
- }
- }
-
-
- /**
- * Runs billing scheduler for given configuration file.
- *
- * @param args the arguments of command line.
- * @throws InitializationException
- */
- public static void startScheduler(String[] args) throws InitializationException {
- if (ServiceUtils.printAppVersion(BillingTool.class, args)) {
- return;
- }
-
- String confName = null;
- for (int i = 0; i < args.length; i++) {
- if (BillingTool.isKey("help", args[i])) {
- i++;
- Help.usage(i < args.length ? Arrays.copyOfRange(args, i, args.length) : null);
- return;
- } else if (BillingTool.isKey("conf", args[i])) {
- i++;
- if (i < args.length) {
- confName = args[i];
- } else {
- throw new InitializationException("Missing the name of configuration file");
- }
- }
- }
-
- if (confName == null) {
- Help.usage();
- throw new InitializationException("Missing arguments");
- }
-
- BillingTool.setLoggerLevel();
- try {
- start(confName);
- } catch (Exception e) {
- throw new DlabException("Billing scheduler failed", e);
- }
- }
-}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingService.java
similarity index 78%
rename from services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java
rename to services/billing-aws/src/main/java/com/epam/dlab/BillingService.java
index f72fa99..9b4d6db 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingService.java
@@ -17,15 +17,12 @@
* under the License.
*/
-package com.epam.dlab.dao;
+package com.epam.dlab;
import com.epam.dlab.dto.billing.BillingData;
import java.util.List;
-public interface BillingDAO {
-
- List<BillingData> getBillingReport(String dateStart, String dateEnd, String dlabId, List<String> products);
-
- List<BillingData> getBillingReport(List<String> dlabIds);
+public interface BillingService {
+ List<BillingData> getBillingData();
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingServiceImpl.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingServiceImpl.java
new file mode 100644
index 0000000..8ac6c48
--- /dev/null
+++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingServiceImpl.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab;
+
+import com.epam.dlab.configuration.BillingToolConfiguration;
+import com.epam.dlab.configuration.BillingToolConfigurationFactory;
+import com.epam.dlab.core.parser.ParserBase;
+import com.epam.dlab.dto.billing.BillingData;
+import com.epam.dlab.exceptions.DlabException;
+import com.epam.dlab.exceptions.InitializationException;
+import com.epam.dlab.util.ServiceUtils;
+import org.bson.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.time.LocalDate;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static com.epam.dlab.model.aws.ReportLine.FIELD_COST;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_CURRENCY_CODE;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_DLAB_ID;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_PRODUCT;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_RESOURCE_TYPE;
+import static com.epam.dlab.model.aws.ReportLine.FIELD_USAGE_DATE;
+
+@Service
+public class BillingServiceImpl implements BillingService {
+ private static final Logger LOGGER = LoggerFactory.getLogger(BillingServiceImpl.class);
+ private static BillingToolConfiguration configuration;
+
+ public List<BillingData> getBillingData() {
+ try {
+ ParserBase parser = configuration.build();
+
+ LOGGER.info("Try to load billing report for configuration: {}", configuration);
+ List<BillingData> billingData = parser.parse()
+ .stream()
+ .map(this::toBillingData)
+ .collect(Collectors.toList());
+
+ if (!parser.getStatistics().isEmpty()) {
+ LOGGER.info("Billing report parser statistics:");
+ for (int i = 0; i < parser.getStatistics().size(); i++) {
+ LOGGER.info(" {}", parser.getStatistics().get(i).toString());
+ }
+ }
+
+ return billingData;
+ } catch (Exception e) {
+ LOGGER.error("Something went wrong ", e);
+ return Collections.emptyList();
+ }
+ }
+
+ private BillingData toBillingData(Document billingData) {
+ return BillingData.builder()
+ .tag(billingData.getString(FIELD_DLAB_ID).toLowerCase())
+ .usageDateFrom(Optional.ofNullable(billingData.getString(FIELD_USAGE_DATE)).map(LocalDate::parse).orElse(null))
+ .usageDateTo(Optional.ofNullable(billingData.getString(FIELD_USAGE_DATE)).map(LocalDate::parse).orElse(null))
+ .usageDate(billingData.getString(FIELD_USAGE_DATE))
+ .product(billingData.getString(FIELD_PRODUCT))
+ .usageType(billingData.getString(FIELD_RESOURCE_TYPE))
+ .cost(billingData.getDouble(FIELD_COST))
+ .currency(billingData.getString(FIELD_CURRENCY_CODE))
+ .build();
+ }
+
+ public static void initialize(String filename) throws InitializationException {
+ LOGGER.debug("Billing report configuration file: {}", filename);
+ configuration = BillingToolConfigurationFactory.build(filename, BillingToolConfiguration.class);
+ }
+
+ public static void startApplication(String[] args) throws InitializationException {
+ if (ServiceUtils.printAppVersion(BillingTool.class, args)) {
+ return;
+ }
+
+ String confName = null;
+ for (int i = 0; i < args.length; i++) {
+ if (BillingTool.isKey("help", args[i])) {
+ i++;
+ Help.usage(i < args.length ? Arrays.copyOfRange(args, i, args.length) : null);
+ return;
+ } else if (BillingTool.isKey("conf", args[i])) {
+ i++;
+ if (i < args.length) {
+ confName = args[i];
+ } else {
+ throw new InitializationException("Missing the name of configuration file");
+ }
+ }
+ }
+
+ if (confName == null) {
+ Help.usage();
+ throw new InitializationException("Missing arguments");
+ }
+
+ BillingTool.setLoggerLevel();
+ try {
+ initialize(confName);
+ } catch (Exception e) {
+ throw new DlabException("Billing scheduler failed", e);
+ }
+ }
+}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/BillingTool.java b/services/billing-aws/src/main/java/com/epam/dlab/BillingTool.java
index cf2b8d6..cde9d4e 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/BillingTool.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/BillingTool.java
@@ -19,24 +19,22 @@
package com.epam.dlab;
-import java.util.Arrays;
-
-import com.epam.dlab.exceptions.DlabException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.LoggerContext;
import com.epam.dlab.configuration.BillingToolConfiguration;
import com.epam.dlab.configuration.BillingToolConfigurationFactory;
import com.epam.dlab.core.parser.ParserBase;
import com.epam.dlab.exceptions.AdapterException;
+import com.epam.dlab.exceptions.DlabException;
import com.epam.dlab.exceptions.InitializationException;
import com.epam.dlab.exceptions.ParseException;
import com.epam.dlab.util.ServiceUtils;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.LoggerContext;
+import java.util.Arrays;
/** Provides billing parser features.
*/
@@ -110,14 +108,14 @@
* @throws InitializationException
*/
public static void main(String[] args) throws InitializationException {
- if (ServiceUtils.printAppVersion(BillingScheduler.class, args)) {
+ if (ServiceUtils.printAppVersion(BillingServiceImpl.class, args)) {
return;
}
String confName = null;
String json = null;
-
- for(int i = 0; i < args.length; i++) {
+
+ for (int i = 0; i < args.length; i++) {
if (isKey("help", args[i])) {
i++;
Help.usage(i < args.length ? Arrays.copyOfRange(args, i, args.length) : null);
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/Help.java b/services/billing-aws/src/main/java/com/epam/dlab/Help.java
index 2a043c2..c2fe5c2 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/Help.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/Help.java
@@ -19,18 +19,17 @@
package com.epam.dlab;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.StringUtils;
-
import com.epam.dlab.core.BillingUtils;
import com.epam.dlab.core.ModuleType;
import com.epam.dlab.exceptions.InitializationException;
import com.fasterxml.jackson.annotation.JsonClassDescription;
import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/** Print help for billing tool.
*/
@@ -47,12 +46,12 @@
private static void printHelp(String resourceName, Map<String, String> substitute) throws InitializationException {
List<String> list = BillingUtils.getResourceAsList("/" + Help.class.getName() + "." + resourceName + ".txt");
String help = StringUtils.join(list, System.lineSeparator());
-
+
if (substitute == null) {
substitute = new HashMap<>();
}
- substitute.put("classname", BillingScheduler.class.getName());
-
+ substitute.put("classname", BillingServiceImpl.class.getName());
+
for (String key : substitute.keySet()) {
help = StringUtils.replace(help, "${" + key.toUpperCase() + "}", substitute.get(key));
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/configuration/BillingToolConfiguration.java b/services/billing-aws/src/main/java/com/epam/dlab/configuration/BillingToolConfiguration.java
index 803d232..420b9e0 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/configuration/BillingToolConfiguration.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/configuration/BillingToolConfiguration.java
@@ -77,13 +77,6 @@
private boolean billingEnabled;
/**
- * Working data file name of modules.
- */
- @Valid
- @JsonProperty
- private SchedulerConfiguration scheduler = null;
-
- /**
* Adapter for reading source data.
*/
@Valid
@@ -136,20 +129,6 @@
}
/**
- * Set the scheduler.
- */
- public void setScheduler(SchedulerConfiguration scheduler) {
- this.scheduler = scheduler;
- }
-
- /**
- * Return the scheduler.
- */
- public SchedulerConfiguration getScheduler() {
- return scheduler;
- }
-
- /**
* Set the adapter for reading source data.
*/
public void setAdapterIn(ImmutableList<AdapterBase> adapter) {
@@ -272,14 +251,6 @@
f.setModuleData(moduleData);
}
- if (scheduler != null) {
- try {
- scheduler.build();
- } catch (Exception e) {
- throw new InitializationException("Cannot configure billing scheduler. " + e.getLocalizedMessage(), e);
- }
- }
-
return parser.build(in, out, f);
}
@@ -295,7 +266,6 @@
public ToStringHelper toStringHelper(Object self) {
return MoreObjects.toStringHelper(self)
.add("moduleData", moduleData)
- .add("scheduler", scheduler)
.add("adapterIn", adapterIn)
.add("adapterOut", adapterOut)
.add("filter", filter)
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/configuration/SchedulerConfiguration.java b/services/billing-aws/src/main/java/com/epam/dlab/configuration/SchedulerConfiguration.java
deleted file mode 100644
index b0624d6..0000000
--- a/services/billing-aws/src/main/java/com/epam/dlab/configuration/SchedulerConfiguration.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.configuration;
-
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.Map;
-import java.util.TreeMap;
-
-import org.apache.commons.lang3.StringUtils;
-
-import com.epam.dlab.exceptions.ParseException;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.MoreObjects;
-import com.google.common.base.MoreObjects.ToStringHelper;
-
-/** Provides schedule time configuration.
- */
-public class SchedulerConfiguration {
-
- /** User's schedule. */
- @JsonProperty
- private String schedule = "12, 13:30:23, 18:34, 08:50, 7:80";
-
-
- /** Return the schedule of user.
- */
- public String getSchedule() {
- return schedule;
- }
-
- /** Set the schedule of user.
- */
- public void setSchedule(String schedule) {
- this.schedule = schedule;
- }
-
-
- /** Schedule. */
- private Map<String, Calendar> realSchedule = new TreeMap<>();
-
- /** Build the schedule from user' schedule.
- * @throws ParseException
- */
- public void build() throws ParseException {
- SimpleDateFormat df = new SimpleDateFormat("HH:mm:ss");
- String [] unitArray = schedule.split(",");
- realSchedule.clear();
- for (int i = 0; i < unitArray.length; i++) {
- Calendar date = Calendar.getInstance();
- int [] time = getTime(unitArray[i]);
- try {
- df.parse(StringUtils.join(time, ':'));
- } catch (Exception e) {
- throw new ParseException("Cannot parse date " + unitArray[i] + ". " + e.getLocalizedMessage(), e);
- }
- date.clear();
- date.set(1, 1, 1, time[0], time[1], time[2]);
- realSchedule.put(df.format(date.getTime()), date);
- }
- adjustStartTime();
- }
-
- /** Return the schedule.
- */
- public Map<String, Calendar> getRealSchedule() {
- return realSchedule;
- }
-
- /** Return time array of user' schedule time.
- * @param time the time in format HH:mm:ss.
- * @throws ParseException
- */
- private int [] getTime(String time) throws ParseException {
- String [] timeString = time.trim().split(":");
- int [] timeInt = new int[3];
-
- for (int i = 0; i < timeInt.length; i++) {
- if (i < timeString.length) {
- try {
- timeInt[i] = Integer.parseInt(timeString[i]);
- } catch (Exception e) {
- throw new ParseException("Cannot parse date " + time + ". " + e.getLocalizedMessage(), e);
- }
- } else {
- timeInt[i] = 0;
- }
- }
-
- return timeInt;
- }
-
- /** Adjust the time in schedule for current time.
- */
- public void adjustStartTime() {
- Calendar now = Calendar.getInstance();
- for(String key : realSchedule.keySet()) {
- Calendar time = realSchedule.get(key);
- if (time.before(now)) {
- time.set(now.get(Calendar.YEAR),
- now.get(Calendar.MONTH),
- now.get(Calendar.DAY_OF_MONTH),
- time.get(Calendar.HOUR_OF_DAY),
- time.get(Calendar.MINUTE),
- time.get(Calendar.SECOND));
- if (time.before(now)) {
- time.add(Calendar.DAY_OF_MONTH, 1);
- }
- realSchedule.put(key, time);
- }
- }
- }
-
- /** Return the key of the next start time from the schedule.
- */
- public String getNextTimeKey() {
- long now = System.currentTimeMillis();
- String nextKey = null;
- long nextTime = -1;
-
- for(String key : realSchedule.keySet()) {
- long time = realSchedule.get(key).getTimeInMillis();
- if ((time >= now && time < nextTime) || nextTime == -1) {
- nextTime = time;
- nextKey = key;
- }
- }
- return nextKey;
- }
-
- /** Return the next start time from the schedule.
- */
- public Calendar getNextTime() {
- String key = getNextTimeKey();
- return (key == null ? null : realSchedule.get(key));
- }
-
- /** Return the key of the near start time from the schedule to the current time.
- */
- public String getNearTimeKey() {
- long now = System.currentTimeMillis();
- String nextKey = null;
- long nextTime = -1;
-
- for(String key : realSchedule.keySet()) {
- long time = Math.abs(now - realSchedule.get(key).getTimeInMillis());
- if (time < nextTime || nextTime == -1) {
- nextTime = time;
- nextKey = key;
- }
- }
- return nextKey;
- }
-
- /** Return the near start time from the schedule to the current time.
- */
- public Calendar getNearTime() {
- String key = getNearTimeKey();
- return (key == null ? null : realSchedule.get(key));
- }
-
- /** Returns a string representation of the object.
- * @param self the object to generate the string for (typically this), used only for its class name.
- */
- public ToStringHelper toStringHelper(Object self) {
- SimpleDateFormat df = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss");
- ToStringHelper helper = MoreObjects.toStringHelper(self);
- for(String key : realSchedule.keySet()) {
- Calendar time = realSchedule.get(key);
- helper.add(key, df.format(time.getTime()));
- }
- return helper;
- }
-
- @Override
- public String toString() {
- return toStringHelper(this)
- .toString();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof SchedulerConfiguration)) return false;
-
- SchedulerConfiguration that = (SchedulerConfiguration) o;
-
- return getRealSchedule() != null ? getRealSchedule().keySet().equals(that.getRealSchedule().keySet())
- : that.getRealSchedule() == null;
- }
-
- @Override
- public int hashCode() {
- return getRealSchedule() != null ? getRealSchedule().keySet().hashCode() : 0;
- }
-}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java b/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java
index 8f70083..deabf44 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/controller/BillingController.java
@@ -19,12 +19,11 @@
package com.epam.dlab.controller;
-import com.epam.dlab.dao.BillingDAO;
+import com.epam.dlab.BillingService;
import com.epam.dlab.dto.billing.BillingData;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
@@ -32,22 +31,14 @@
@RestController
public class BillingController {
- private final BillingDAO billingDAO;
+ private final BillingService billingService;
- public BillingController(BillingDAO billingDAO) {
- this.billingDAO = billingDAO;
+ public BillingController(BillingService billingService) {
+ this.billingService = billingService;
}
@GetMapping
- public ResponseEntity<List<BillingData>> getBilling(@RequestParam List<String> dlabIds) {
- return new ResponseEntity<>(billingDAO.getBillingReport(dlabIds), HttpStatus.OK);
- }
-
- @GetMapping("/report")
- public ResponseEntity<List<BillingData>> getBilling(@RequestParam("date-start") String dateStart,
- @RequestParam("date-end") String dateEnd,
- @RequestParam("dlab-id") String dlabId,
- @RequestParam("product") List<String> products) {
- return new ResponseEntity<>(billingDAO.getBillingReport(dateStart, dateEnd, dlabId, products), HttpStatus.OK);
+ public ResponseEntity<List<BillingData>> getBilling() {
+ return new ResponseEntity<>(billingService.getBillingData(), HttpStatus.OK);
}
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/core/AdapterBase.java b/services/billing-aws/src/main/java/com/epam/dlab/core/AdapterBase.java
index 1569530..475404d 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/core/AdapterBase.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/core/AdapterBase.java
@@ -24,6 +24,7 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects.ToStringHelper;
+import org.bson.Document;
import java.util.List;
@@ -157,9 +158,10 @@
* Write the row of data to adapter.
*
* @param row the row of common format.
+ * @return
* @throws AdapterException
*/
- public abstract void writeRow(ReportLine row) throws AdapterException;
+ public abstract Document writeRow(ReportLine row) throws AdapterException;
@Override
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/core/parser/ParserBase.java b/services/billing-aws/src/main/java/com/epam/dlab/core/parser/ParserBase.java
index f9f0eaa..bfd86bc 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/core/parser/ParserBase.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/core/parser/ParserBase.java
@@ -19,13 +19,6 @@
package com.epam.dlab.core.parser;
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.validation.constraints.NotNull;
-
-import org.apache.commons.lang3.StringUtils;
-
import com.epam.dlab.core.AdapterBase;
import com.epam.dlab.core.FilterBase;
import com.epam.dlab.core.ModuleBase;
@@ -37,6 +30,12 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.MoreObjects.ToStringHelper;
+import org.apache.commons.lang3.StringUtils;
+import org.bson.Document;
+
+import javax.validation.constraints.NotNull;
+import java.util.ArrayList;
+import java.util.List;
/** Abstract module of parser.<br>
* See description of {@link ModuleBase} how to create your own parser.
@@ -234,13 +233,16 @@
* @throws InitializationException
*/
public abstract void initialize() throws InitializationException;
-
- /** Parse the source data to common format and write it to output adapter.
+
+ /**
+ * Parse the source data to common format and write it to output adapter.
+ *
+ * @return
* @throws InitializationException
* @throws AdapterException
* @throws ParseException
*/
- public abstract void parse() throws InitializationException, AdapterException, ParseException;
+ public abstract List<Document> parse() throws InitializationException, AdapterException, ParseException;
/** Build parser from given modules.
* @param adapterIn the adapter for reading source data.
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/core/parser/ParserByLine.java b/services/billing-aws/src/main/java/com/epam/dlab/core/parser/ParserByLine.java
index 37f2070..d878cb9 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/core/parser/ParserByLine.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/core/parser/ParserByLine.java
@@ -27,10 +27,12 @@
import com.epam.dlab.exceptions.ParseException;
import com.epam.dlab.model.aws.ReportLine;
import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -156,11 +158,13 @@
/**
* Parse the source data to common format and write it to output adapter.
*
+ * @return list of billing data
* @throws InitializationException
* @throws AdapterException
* @throws ParseException
*/
- public void parse() throws InitializationException, AdapterException, ParseException {
+ public List<Document> parse() throws InitializationException, AdapterException, ParseException {
+ List<Document> billingData = new ArrayList<>();
try {
if (init()) {
String line;
@@ -211,14 +215,14 @@
if (getAggregate() != AggregateGranularity.NONE) {
getAggregator().append(reportLine);
} else {
- getAdapterOut().writeRow(reportLine);
+ billingData.add(getAdapterOut().writeRow(reportLine));
getCurrentStatistics().incrRowWritten();
}
}
if (getAggregate() != AggregateGranularity.NONE) {
for (int i = 0; i < getAggregator().size(); i++) {
- getAdapterOut().writeRow(getAggregator().get(i));
+ billingData.add(getAdapterOut().writeRow(getAggregator().get(i)));
getCurrentStatistics().incrRowWritten();
}
}
@@ -255,5 +259,6 @@
if (getCurrentStatistics() != null) {
getCurrentStatistics().stop();
}
+ return billingData;
}
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java b/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java
deleted file mode 100644
index b0ff9f1..0000000
--- a/services/billing-aws/src/main/java/com/epam/dlab/dao/impl/BillingDAOImpl.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.dao.impl;
-
-import com.epam.dlab.dao.BillingDAO;
-import com.epam.dlab.dto.billing.BillingData;
-import com.epam.dlab.exceptions.DlabException;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.bson.Document;
-import org.springframework.data.mongodb.core.MongoTemplate;
-import org.springframework.data.mongodb.core.aggregation.Aggregation;
-import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
-import org.springframework.data.mongodb.core.aggregation.GroupOperation;
-import org.springframework.data.mongodb.core.aggregation.MatchOperation;
-import org.springframework.data.mongodb.core.query.Criteria;
-import org.springframework.stereotype.Component;
-
-import java.math.BigDecimal;
-import java.time.LocalDate;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static com.epam.dlab.model.aws.ReportLine.FIELD_COST;
-import static com.epam.dlab.model.aws.ReportLine.FIELD_CURRENCY_CODE;
-import static com.epam.dlab.model.aws.ReportLine.FIELD_DLAB_ID;
-import static com.epam.dlab.model.aws.ReportLine.FIELD_PRODUCT;
-import static com.epam.dlab.model.aws.ReportLine.FIELD_RESOURCE_TYPE;
-import static com.epam.dlab.model.aws.ReportLine.FIELD_USAGE_DATE;
-import static org.springframework.data.mongodb.core.aggregation.Aggregation.group;
-import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation;
-
-@Component
-@Slf4j
-public class BillingDAOImpl implements BillingDAO {
- private final MongoTemplate mongoTemplate;
-
- public BillingDAOImpl(MongoTemplate mongoTemplate) {
- this.mongoTemplate = mongoTemplate;
- }
-
- @Override
- public List<BillingData> getBillingReport(String dateStart, String dateEnd, String dlabId, List<String> products) {
- try {
- List<AggregationOperation> aggregationOperations = new ArrayList<>();
- aggregationOperations.add(Aggregation.match(Criteria.where(FIELD_DLAB_ID).regex(dlabId, "i")));
- if (!products.isEmpty()) {
- aggregationOperations.add(Aggregation.match(Criteria.where(FIELD_PRODUCT).in(products)));
- }
- getMatchCriteria(dateStart, Criteria.where(FIELD_USAGE_DATE).gte(dateStart))
- .ifPresent(aggregationOperations::add);
- getMatchCriteria(dateEnd, Criteria.where(FIELD_USAGE_DATE).lte(dateEnd))
- .ifPresent(aggregationOperations::add);
- aggregationOperations.add(getGroupOperation());
-
- Aggregation aggregation = newAggregation(aggregationOperations);
-
- return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults()
- .stream()
- .map(this::toBillingData)
- .collect(Collectors.toList());
- } catch (Exception e) {
- log.error("Cannot retrieve billing information ", e);
- throw new DlabException("Cannot retrieve billing information", e);
- }
- }
-
- @Override
- public List<BillingData> getBillingReport(List<String> dlabIds) {
- try {
- GroupOperation groupOperation = getGroupOperation();
- MatchOperation matchOperation = Aggregation.match(Criteria.where("dlab_id").in(dlabIds));
- Aggregation aggregation = newAggregation(matchOperation, groupOperation);
-
- return mongoTemplate.aggregate(aggregation, "billing", Document.class).getMappedResults()
- .stream()
- .map(this::toBillingData)
- .collect(Collectors.toList());
- } catch (Exception e) {
- log.error("Cannot retrieve billing information ", e);
- throw new DlabException("Cannot retrieve billing information", e);
- }
- }
-
- private GroupOperation getGroupOperation() {
- return group(FIELD_PRODUCT, FIELD_CURRENCY_CODE, FIELD_RESOURCE_TYPE, FIELD_DLAB_ID)
- .min(FIELD_USAGE_DATE).as("from")
- .max(FIELD_USAGE_DATE).as("to")
- .sum(FIELD_COST).as(FIELD_COST);
- }
-
- private Optional<MatchOperation> getMatchCriteria(String dateStart, Criteria criteria) {
- return Optional.ofNullable(dateStart)
- .filter(StringUtils::isNotEmpty)
- .map(date -> Aggregation.match(criteria));
- }
-
- private BillingData toBillingData(Document billingData) {
- return BillingData.builder()
- .tag(billingData.getString(FIELD_DLAB_ID))
- .usageDateFrom(Optional.ofNullable(billingData.getString("from")).map(LocalDate::parse).orElse(null))
- .usageDateTo(Optional.ofNullable(billingData.getString("to")).map(LocalDate::parse).orElse(null))
- .product(billingData.getString(FIELD_PRODUCT))
- .usageType(billingData.getString(FIELD_RESOURCE_TYPE))
- .cost(BigDecimal.valueOf(billingData.getDouble(FIELD_COST)).setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue())
- .currency(billingData.getString(FIELD_CURRENCY_CODE))
- .build();
- }
-}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/module/AdapterConsole.java b/services/billing-aws/src/main/java/com/epam/dlab/module/AdapterConsole.java
index 59c866d..3bffa79 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/module/AdapterConsole.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/module/AdapterConsole.java
@@ -25,6 +25,7 @@
import com.epam.dlab.model.aws.ReportLine;
import com.fasterxml.jackson.annotation.JsonClassDescription;
import com.fasterxml.jackson.annotation.JsonTypeName;
+import org.bson.Document;
import java.util.List;
@@ -84,7 +85,8 @@
}
@Override
- public void writeRow(ReportLine row) throws AdapterException {
+ public Document writeRow(ReportLine row) throws AdapterException {
System.out.println(CommonFormat.rowToString(row));
+ return null;
}
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/module/AdapterFile.java b/services/billing-aws/src/main/java/com/epam/dlab/module/AdapterFile.java
index 7fb38f3..dd256eb 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/module/AdapterFile.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/module/AdapterFile.java
@@ -19,15 +19,6 @@
package com.epam.dlab.module;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.List;
-
-import javax.validation.constraints.NotNull;
-
import com.epam.dlab.core.AdapterBase;
import com.epam.dlab.core.parser.CommonFormat;
import com.epam.dlab.exceptions.AdapterException;
@@ -37,6 +28,15 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.base.MoreObjects.ToStringHelper;
+import org.bson.Document;
+
+import javax.validation.constraints.NotNull;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
/** The adapter for file system.
*/
@@ -137,15 +137,16 @@
throw new AdapterException("Cannot write file " + file + ". " + e.getLocalizedMessage(), e);
}
}
-
+
@Override
- public void writeRow(ReportLine row) throws AdapterException {
+ public Document writeRow(ReportLine row) throws AdapterException {
try {
writer.write(CommonFormat.rowToString(row));
writer.write(System.lineSeparator());
} catch (IOException e) {
throw new AdapterException("Cannot write file " + file + ". " + e.getLocalizedMessage(), e);
}
+ return null;
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/module/aws/AdapterS3File.java b/services/billing-aws/src/main/java/com/epam/dlab/module/aws/AdapterS3File.java
index 9dc7e07..0579063 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/module/aws/AdapterS3File.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/module/aws/AdapterS3File.java
@@ -33,6 +33,7 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeName;
import com.google.common.base.MoreObjects.ToStringHelper;
+import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -308,7 +309,7 @@
}
@Override
- public void writeRow(ReportLine row) throws AdapterException {
+ public Document writeRow(ReportLine row) throws AdapterException {
throw new AdapterException("Unimplemented method.");
}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java b/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java
index 994522d..db92a80 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java
+++ b/services/billing-aws/src/main/java/com/epam/dlab/mongo/AdapterMongoDb.java
@@ -219,7 +219,7 @@
}
@Override
- public void writeRow(ReportLine row) throws AdapterException {
+ public Document writeRow(ReportLine row) throws AdapterException {
Document document;
try {
document = resourceTypeDAO.transform(row);
@@ -227,20 +227,21 @@
throw new AdapterException("Cannot transform report line. " + e.getLocalizedMessage(), e);
}
- usageDateList.append(row.getUsageDate());
- if (upsert) {
- buffer.add(document);
- if (buffer.size() >= bufferSize) {
- connection.upsertRows(collection, buffer, usageDateList);
- }
- } else if (bufferSize > 0) {
- buffer.add(document);
- if (buffer.size() >= bufferSize) {
- connection.insertRows(collection, buffer);
- }
- } else {
- connection.insertOne(collection, document);
- }
+// usageDateList.append(row.getUsageDate());
+// if (upsert) {
+// buffer.add(document);
+// if (buffer.size() >= bufferSize) {
+// connection.upsertRows(collection, buffer, usageDateList);
+// }
+// } else if (bufferSize > 0) {
+// buffer.add(document);
+// if (buffer.size() >= bufferSize) {
+// connection.insertRows(collection, buffer);
+// }
+// } else {
+// connection.insertOne(collection, document);
+// }
+ return document;
}
/**
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
deleted file mode 100644
index f81fe2e..0000000
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/BillingSchedulerAzure.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.billing.azure;
-
-import com.epam.dlab.MongoKeyWords;
-import com.epam.dlab.billing.azure.config.AzureAuthFile;
-import com.epam.dlab.billing.azure.config.BillingConfigurationAzure;
-import com.epam.dlab.billing.azure.model.AzureDailyResourceInvoice;
-import com.epam.dlab.billing.azure.model.BillingPeriod;
-import com.epam.dlab.exceptions.DlabException;
-import com.epam.dlab.util.mongo.modules.IsoDateModule;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.mongodb.BasicDBObject;
-import com.mongodb.client.model.Filters;
-import com.mongodb.client.model.UpdateOptions;
-import com.mongodb.client.result.UpdateResult;
-import lombok.extern.slf4j.Slf4j;
-import org.bson.Document;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.PostConstruct;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-@Slf4j
-@Component
-public class BillingSchedulerAzure {
- private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
- private BillingConfigurationAzure billingConfigurationAzure;
- private MongoDbBillingClient mongoDbBillingClient;
-
- @Autowired
- public BillingSchedulerAzure(BillingConfigurationAzure configuration) throws IOException {
- billingConfigurationAzure = configuration;
- Path path = Paths.get(billingConfigurationAzure.getAuthenticationFile());
-
- if (path.toFile().exists()) {
- log.info("Read and override configs using auth file");
- try {
- AzureAuthFile azureAuthFile = new ObjectMapper().readValue(path.toFile(), AzureAuthFile.class);
- this.billingConfigurationAzure.setClientId(azureAuthFile.getClientId());
- this.billingConfigurationAzure.setClientSecret(azureAuthFile.getClientSecret());
- this.billingConfigurationAzure.setTenantId(azureAuthFile.getTenantId());
- this.billingConfigurationAzure.setSubscriptionId(azureAuthFile.getSubscriptionId());
- } catch (IOException e) {
- log.error("Cannot read configuration file", e);
- throw e;
- }
- log.info("Configs from auth file are used");
- } else {
- log.info("Configs from yml file are used");
- }
-
- this.mongoDbBillingClient = new MongoDbBillingClient
- (billingConfigurationAzure.getAggregationOutputMongoDataSource().getHost(),
- billingConfigurationAzure.getAggregationOutputMongoDataSource().getPort(),
- billingConfigurationAzure.getAggregationOutputMongoDataSource().getDatabase(),
- billingConfigurationAzure.getAggregationOutputMongoDataSource().getUsername(),
- billingConfigurationAzure.getAggregationOutputMongoDataSource().getPassword());
- }
-
- @PostConstruct
- public void start() {
- if (billingConfigurationAzure.isBillingEnabled()) {
- executorService.scheduleWithFixedDelay(new CalculateBilling(billingConfigurationAzure,
- mongoDbBillingClient), billingConfigurationAzure.getInitialDelay(),
- billingConfigurationAzure.getPeriod(), TimeUnit.MINUTES);
- } else {
- log.info("======Billing is disabled======");
- }
- }
-
- public void stop() {
- try {
- log.info("Stopping Azure billing scheduler");
- if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) {
- log.error("Force shut down");
- executorService.shutdownNow();
- }
- mongoDbBillingClient.getClient().close();
- } catch (InterruptedException e) {
- executorService.shutdownNow();
- mongoDbBillingClient.getClient().close();
- Thread.currentThread().interrupt();
- }
- }
-
-
- @Slf4j
- private static class CalculateBilling implements Runnable {
- private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern
- ("yyyy-MM-dd'T'HH:mm:ss.SSS'Z");
- private static final String SCHEDULER_ID = "azureBillingScheduler";
- private BillingConfigurationAzure billingConfigurationAzure;
- private MongoDbBillingClient client;
- private ObjectMapper objectMapper = new ObjectMapper().registerModule(new IsoDateModule());
-
-
- public CalculateBilling(BillingConfigurationAzure billingConfigurationAzure, MongoDbBillingClient client) {
- this.billingConfigurationAzure = billingConfigurationAzure;
- this.client = client;
- }
-
- @Override
- public void run() {
- try {
- BillingPeriod billingPeriod = getBillingPeriod();
- DateTime currentTime = new DateTime().withZone(DateTimeZone.UTC);
- if (billingPeriod == null) {
- saveBillingPeriod(initialSchedulerInfo(currentTime));
- } else {
- log.info("Billing period from db is {}", billingPeriod);
-
- if (shouldTriggerJobByTime(currentTime, billingPeriod)) {
- boolean hasNew = run(billingPeriod);
- updateBillingPeriod(billingPeriod, currentTime, hasNew);
- }
- }
- } catch (RuntimeException e) {
- log.error("Cannot update billing information", e);
- }
- }
-
- private BillingPeriod initialSchedulerInfo(DateTime currentTime) {
-
- BillingPeriod initialBillingPeriod = new BillingPeriod();
- initialBillingPeriod.setFrom(currentTime.minusDays(2).toDateMidnight().toDate());
- initialBillingPeriod.setTo(currentTime.toDateMidnight().toDate());
-
- log.info("Initial scheduler info {}", initialBillingPeriod);
-
- return initialBillingPeriod;
-
- }
-
- private boolean shouldTriggerJobByTime(DateTime currentTime, BillingPeriod billingPeriod) {
-
- DateTime dateTimeToFromBillingPeriod = new DateTime(billingPeriod.getTo()).withZone(DateTimeZone.UTC);
-
- log.info("Comparing current time[{}, {}] and from scheduler info [{}, {}]", currentTime,
- currentTime.toDateMidnight(),
- dateTimeToFromBillingPeriod, dateTimeToFromBillingPeriod.toDateMidnight());
-
- if (currentTime.toDateMidnight().isAfter(dateTimeToFromBillingPeriod.toDateMidnight())
- || currentTime.toDateMidnight().isEqual(dateTimeToFromBillingPeriod.toDateMidnight())) {
- log.info("Should trigger the job by time");
- return true;
- }
-
- log.info("Should not trigger the job by time");
- return false;
- }
-
- private boolean run(BillingPeriod billingPeriod) {
- AzureInvoiceCalculationService azureInvoiceCalculationService
- = new AzureInvoiceCalculationService(billingConfigurationAzure);
-
- List<AzureDailyResourceInvoice> dailyInvoices = azureInvoiceCalculationService.generateInvoiceData(
- DATE_TIME_FORMATTER.print(new DateTime(billingPeriod.getFrom()).withZone(DateTimeZone.UTC)),
- DATE_TIME_FORMATTER.print(new DateTime(billingPeriod.getTo()).withZone(DateTimeZone.UTC)));
-
- if (!dailyInvoices.isEmpty()) {
- client.getDatabase().getCollection(MongoKeyWords.BILLING_DETAILS)
- .insertMany(dailyInvoices.stream().map(AzureDailyResourceInvoice::to)
- .collect(Collectors.toList()));
- return true;
- } else {
- log.warn("Daily invoices is empty for period {}", billingPeriod);
- return false;
- }
- }
-
- private void updateBillingPeriod(BillingPeriod billingPeriod, DateTime currentTime, boolean updates) {
-
- try {
- client.getDatabase().getCollection(MongoKeyWords.AZURE_BILLING_SCHEDULER_HISTORY).insertOne(
- Document.parse(objectMapper.writeValueAsString(billingPeriod)).append("updates", updates));
- log.debug("History of billing periods is updated with {}",
- objectMapper.writeValueAsString(billingPeriod));
- } catch (JsonProcessingException e) {
- log.error("Cannot update history of billing periods", e);
-
- }
-
- billingPeriod.setFrom(billingPeriod.getTo());
-
- if (new DateTime(billingPeriod.getFrom()).withZone(DateTimeZone.UTC).toDateMidnight()
- .isEqual(currentTime.toDateMidnight())) {
-
- log.info("Setting billing to one day later");
- billingPeriod.setTo(currentTime.plusDays(1).toDateMidnight().toDate());
-
- } else {
- billingPeriod.setTo(currentTime.toDateMidnight().toDate());
- }
-
- saveBillingPeriod(billingPeriod);
- }
-
- private boolean saveBillingPeriod(BillingPeriod billingPeriod) {
- log.debug("Saving billing period {}", billingPeriod);
-
- try {
- UpdateResult updateResult = client.getDatabase().getCollection(MongoKeyWords.AZURE_BILLING_SCHEDULER)
- .updateMany(Filters.eq(MongoKeyWords.MONGO_ID, SCHEDULER_ID),
- new BasicDBObject("$set",
- Document.parse(objectMapper.writeValueAsString(billingPeriod))
- .append(MongoKeyWords.MONGO_ID, SCHEDULER_ID))
- , new UpdateOptions().upsert(true)
- );
-
- log.debug("Billing period save operation result is {}", updateResult);
- return true;
- } catch (JsonProcessingException e) {
- log.error("Cannot save billing period", e);
- }
-
- return false;
- }
-
- private BillingPeriod getBillingPeriod() {
- log.debug("Get billing period");
-
- try {
- Document document = client.getDatabase().getCollection(MongoKeyWords.AZURE_BILLING_SCHEDULER)
- .find(Filters.eq(MongoKeyWords.MONGO_ID, SCHEDULER_ID)).first();
-
- log.debug("Retrieved billing period document {}", document);
- if (document != null) {
- return objectMapper.readValue(document.toJson(), BillingPeriod.class);
- }
-
- return null;
-
- } catch (IOException e) {
- log.error("Cannot save billing period", e);
- throw new DlabException("Cannot parse string", e);
- }
- }
- }
-}
diff --git a/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingService.java
similarity index 78%
copy from services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java
copy to services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingService.java
index f72fa99..d432337 100644
--- a/services/billing-aws/src/main/java/com/epam/dlab/dao/BillingDAO.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingService.java
@@ -17,15 +17,12 @@
* under the License.
*/
-package com.epam.dlab.dao;
+package com.epam.dlab.billing.azure;
import com.epam.dlab.dto.billing.BillingData;
import java.util.List;
-public interface BillingDAO {
-
- List<BillingData> getBillingReport(String dateStart, String dateEnd, String dlabId, List<String> products);
-
- List<BillingData> getBillingReport(List<String> dlabIds);
+public interface CalculateBillingService {
+ List<BillingData> getBillingData();
}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java
new file mode 100644
index 0000000..3b3d60b
--- /dev/null
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/CalculateBillingServiceImpl.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.billing.azure;
+
+import com.epam.dlab.MongoKeyWords;
+import com.epam.dlab.billing.azure.config.AzureAuthFile;
+import com.epam.dlab.billing.azure.config.BillingConfigurationAzure;
+import com.epam.dlab.billing.azure.model.AzureDailyResourceInvoice;
+import com.epam.dlab.billing.azure.model.BillingPeriod;
+import com.epam.dlab.dto.billing.BillingData;
+import com.epam.dlab.exceptions.DlabException;
+import com.epam.dlab.util.mongo.modules.IsoDateModule;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.mongodb.BasicDBObject;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.UpdateOptions;
+import com.mongodb.client.result.UpdateResult;
+import lombok.extern.slf4j.Slf4j;
+import org.bson.Document;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Slf4j
+@Service
+public class CalculateBillingServiceImpl implements CalculateBillingService {
+ private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z");
+ private static final String SCHEDULER_ID = "azureBillingScheduler";
+ private final BillingConfigurationAzure billingConfigurationAzure;
+ private final MongoDbBillingClient mongoDbBillingClient;
+ private ObjectMapper objectMapper;
+
+ @Autowired
+ public CalculateBillingServiceImpl(BillingConfigurationAzure configuration) throws IOException {
+ billingConfigurationAzure = configuration;
+ objectMapper = new ObjectMapper().registerModule(new IsoDateModule());
+ Path path = Paths.get(billingConfigurationAzure.getAuthenticationFile());
+
+ if (path.toFile().exists()) {
+ log.info("Read and override configs using auth file");
+ try {
+ AzureAuthFile azureAuthFile = new ObjectMapper().readValue(path.toFile(), AzureAuthFile.class);
+ this.billingConfigurationAzure.setClientId(azureAuthFile.getClientId());
+ this.billingConfigurationAzure.setClientSecret(azureAuthFile.getClientSecret());
+ this.billingConfigurationAzure.setTenantId(azureAuthFile.getTenantId());
+ this.billingConfigurationAzure.setSubscriptionId(azureAuthFile.getSubscriptionId());
+ } catch (IOException e) {
+ log.error("Cannot read configuration file", e);
+ throw e;
+ }
+ log.info("Configs from auth file are used");
+ } else {
+ log.info("Configs from yml file are used");
+ }
+
+ this.mongoDbBillingClient = new MongoDbBillingClient
+ (billingConfigurationAzure.getAggregationOutputMongoDataSource().getHost(),
+ billingConfigurationAzure.getAggregationOutputMongoDataSource().getPort(),
+ billingConfigurationAzure.getAggregationOutputMongoDataSource().getDatabase(),
+ billingConfigurationAzure.getAggregationOutputMongoDataSource().getUsername(),
+ billingConfigurationAzure.getAggregationOutputMongoDataSource().getPassword());
+ }
+
+ @Override
+ public List<BillingData> getBillingData() {
+ try {
+ BillingPeriod billingPeriod = getBillingPeriod();
+ DateTime currentTime = new DateTime().withZone(DateTimeZone.UTC);
+ if (billingPeriod == null) {
+ saveBillingPeriod(initialSchedulerInfo(currentTime));
+ } else {
+ log.info("Billing period from db is {}", billingPeriod);
+
+ if (shouldTriggerJobByTime(currentTime, billingPeriod)) {
+ List<BillingData> billingData = getBillingData(billingPeriod);
+ boolean hasNew = !billingData.isEmpty();
+ updateBillingPeriod(billingPeriod, currentTime, hasNew);
+ return billingData;
+ }
+ }
+ } catch (RuntimeException e) {
+ log.error("Cannot update billing information", e);
+ }
+ return Collections.emptyList();
+ }
+
+ private BillingPeriod initialSchedulerInfo(DateTime currentTime) {
+
+ BillingPeriod initialBillingPeriod = new BillingPeriod();
+ initialBillingPeriod.setFrom(currentTime.minusDays(2).toDateMidnight().toDate());
+ initialBillingPeriod.setTo(currentTime.toDateMidnight().toDate());
+
+ log.info("Initial scheduler info {}", initialBillingPeriod);
+
+ return initialBillingPeriod;
+
+ }
+
+ private boolean shouldTriggerJobByTime(DateTime currentTime, BillingPeriod billingPeriod) {
+
+ DateTime dateTimeToFromBillingPeriod = new DateTime(billingPeriod.getTo()).withZone(DateTimeZone.UTC);
+
+ log.info("Comparing current time[{}, {}] and from scheduler info [{}, {}]", currentTime,
+ currentTime.toDateMidnight(),
+ dateTimeToFromBillingPeriod, dateTimeToFromBillingPeriod.toDateMidnight());
+
+ if (currentTime.toDateMidnight().isAfter(dateTimeToFromBillingPeriod.toDateMidnight())
+ || currentTime.toDateMidnight().isEqual(dateTimeToFromBillingPeriod.toDateMidnight())) {
+ log.info("Should trigger the job by time");
+ return true;
+ }
+
+ log.info("Should not trigger the job by time");
+ return false;
+ }
+
+ private List<BillingData> getBillingData(BillingPeriod billingPeriod) {
+ AzureInvoiceCalculationService azureInvoiceCalculationService
+ = new AzureInvoiceCalculationService(billingConfigurationAzure);
+
+ List<AzureDailyResourceInvoice> dailyInvoices = azureInvoiceCalculationService.generateInvoiceData(
+ DATE_TIME_FORMATTER.print(new DateTime(billingPeriod.getFrom()).withZone(DateTimeZone.UTC)),
+ DATE_TIME_FORMATTER.print(new DateTime(billingPeriod.getTo()).withZone(DateTimeZone.UTC)));
+
+ if (!dailyInvoices.isEmpty()) {
+ return dailyInvoices
+ .stream()
+ .map(this::toBillingData)
+ .collect(Collectors.toList());
+ } else {
+ log.warn("Daily invoices is empty for period {}", billingPeriod);
+ return Collections.emptyList();
+ }
+ }
+
+ private void updateBillingPeriod(BillingPeriod billingPeriod, DateTime currentTime, boolean updates) {
+
+ try {
+ mongoDbBillingClient.getDatabase().getCollection(MongoKeyWords.AZURE_BILLING_SCHEDULER_HISTORY).insertOne(
+ Document.parse(objectMapper.writeValueAsString(billingPeriod)).append("updates", updates));
+ log.debug("History of billing periods is updated with {}",
+ objectMapper.writeValueAsString(billingPeriod));
+ } catch (JsonProcessingException e) {
+ log.error("Cannot update history of billing periods", e);
+
+ }
+
+ billingPeriod.setFrom(billingPeriod.getTo());
+
+ if (new DateTime(billingPeriod.getFrom()).withZone(DateTimeZone.UTC).toDateMidnight()
+ .isEqual(currentTime.toDateMidnight())) {
+
+ log.info("Setting billing to one day later");
+ billingPeriod.setTo(currentTime.plusDays(1).toDateMidnight().toDate());
+
+ } else {
+ billingPeriod.setTo(currentTime.toDateMidnight().toDate());
+ }
+
+ saveBillingPeriod(billingPeriod);
+ }
+
+ private boolean saveBillingPeriod(BillingPeriod billingPeriod) {
+ log.debug("Saving billing period {}", billingPeriod);
+
+ try {
+ UpdateResult updateResult = mongoDbBillingClient.getDatabase().getCollection(MongoKeyWords.AZURE_BILLING_SCHEDULER)
+ .updateMany(Filters.eq(MongoKeyWords.MONGO_ID, SCHEDULER_ID),
+ new BasicDBObject("$set",
+ Document.parse(objectMapper.writeValueAsString(billingPeriod))
+ .append(MongoKeyWords.MONGO_ID, SCHEDULER_ID))
+ , new UpdateOptions().upsert(true)
+ );
+
+ log.debug("Billing period save operation result is {}", updateResult);
+ return true;
+ } catch (JsonProcessingException e) {
+ log.error("Cannot save billing period", e);
+ }
+
+ return false;
+ }
+
+ private BillingPeriod getBillingPeriod() {
+ log.debug("Get billing period");
+
+ try {
+ Document document = mongoDbBillingClient.getDatabase().getCollection(MongoKeyWords.AZURE_BILLING_SCHEDULER)
+ .find(Filters.eq(MongoKeyWords.MONGO_ID, SCHEDULER_ID)).first();
+
+ log.debug("Retrieved billing period document {}", document);
+ if (document != null) {
+ return objectMapper.readValue(document.toJson(), BillingPeriod.class);
+ }
+
+ return null;
+
+ } catch (IOException e) {
+ log.error("Cannot save billing period", e);
+ throw new DlabException("Cannot parse string", e);
+ }
+ }
+
+ private BillingData toBillingData(AzureDailyResourceInvoice billingData) {
+ return BillingData.builder()
+ .tag(billingData.getDlabId().toLowerCase())
+ .usageDateFrom(Optional.ofNullable(billingData.getUsageStartDate()).map(LocalDate::parse).orElse(null))
+ .usageDateTo(Optional.ofNullable(billingData.getUsageEndDate()).map(LocalDate::parse).orElse(null))
+ .usageDate(billingData.getDay())
+ .product(billingData.getMeterCategory())
+ .cost(billingData.getCost())
+ .currency(billingData.getCurrencyCode())
+ .build();
+ }
+}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java
index eb728c9..9018791 100644
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java
+++ b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/controller/BillingController.java
@@ -19,12 +19,11 @@
package com.epam.dlab.billing.azure.controller;
-import com.epam.dlab.billing.azure.dao.BillingDAO;
+import com.epam.dlab.billing.azure.CalculateBillingService;
import com.epam.dlab.dto.billing.BillingData;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
@@ -32,22 +31,14 @@
@RestController
public class BillingController {
- private final BillingDAO billingDAO;
+ private final CalculateBillingService billingService;
- public BillingController(BillingDAO billingDAO) {
- this.billingDAO = billingDAO;
+ public BillingController(CalculateBillingService billingService) {
+ this.billingService = billingService;
}
@GetMapping
- public ResponseEntity<List<BillingData>> getBilling(@RequestParam List<String> dlabIds) {
- return new ResponseEntity<>(billingDAO.getBillingReport(dlabIds), HttpStatus.OK);
- }
-
- @GetMapping("/report")
- public ResponseEntity<List<BillingData>> getBilling(@RequestParam("date-start") String dateStart,
- @RequestParam("date-end") String dateEnd,
- @RequestParam("dlab-id") String dlabId,
- @RequestParam("product") List<String> products) {
- return new ResponseEntity<>(billingDAO.getBillingReport(dateStart, dateEnd, dlabId, products), HttpStatus.OK);
+ public ResponseEntity<List<BillingData>> getBilling() {
+ return new ResponseEntity<>(billingService.getBillingData(), HttpStatus.OK);
}
}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java
deleted file mode 100644
index 793d7cb..0000000
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/BillingDAO.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.billing.azure.dao;
-
-import com.epam.dlab.dto.billing.BillingData;
-
-import java.util.List;
-
-public interface BillingDAO {
-
- List<BillingData> getBillingReport(String dateStart, String dateEnd, String dlabId, List<String> products);
-
- List<BillingData> getBillingReport(List<String> dlabIds);
-}
diff --git a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java b/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java
deleted file mode 100644
index c39385a..0000000
--- a/services/billing-azure/src/main/java/com/epam/dlab/billing/azure/dao/impl/BillingDAOImpl.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.billing.azure.dao.impl;
-
-import com.epam.dlab.billing.azure.dao.BillingDAO;
-import com.epam.dlab.billing.azure.model.AzureDailyResourceInvoice;
-import com.epam.dlab.dto.billing.BillingData;
-import com.epam.dlab.exceptions.DlabException;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
-import org.springframework.data.mongodb.core.MongoTemplate;
-import org.springframework.data.mongodb.core.aggregation.Aggregation;
-import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
-import org.springframework.data.mongodb.core.aggregation.GroupOperation;
-import org.springframework.data.mongodb.core.aggregation.MatchOperation;
-import org.springframework.data.mongodb.core.query.Criteria;
-import org.springframework.stereotype.Component;
-
-import java.math.BigDecimal;
-import java.time.LocalDate;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.stream.Collectors;
-
-import static org.springframework.data.mongodb.core.aggregation.Aggregation.group;
-import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation;
-
-@Component
-@Slf4j
-public class BillingDAOImpl implements BillingDAO {
- private final MongoTemplate mongoTemplate;
-
- public BillingDAOImpl(MongoTemplate mongoTemplate) {
- this.mongoTemplate = mongoTemplate;
- }
-
- @Override
- public List<BillingData> getBillingReport(String dateStart, String dateEnd, String dlabId, List<String> products) {
- try {
- List<AggregationOperation> aggregationOperations = new ArrayList<>();
- aggregationOperations.add(Aggregation.match(Criteria.where("dlabId").regex(dlabId, "i")));
- if (!products.isEmpty()) {
- aggregationOperations.add(Aggregation.match(Criteria.where("meterCategory").in(products)));
- }
- getMatchCriteria(dateStart, Criteria.where("day").gte(dateStart))
- .ifPresent(aggregationOperations::add);
- getMatchCriteria(dateEnd, Criteria.where("day").lte(dateEnd))
- .ifPresent(aggregationOperations::add);
- aggregationOperations.add(getGroupOperation());
-
- Aggregation aggregation = newAggregation(aggregationOperations);
-
- return mongoTemplate.aggregate(aggregation, "billing", AzureDailyResourceInvoice.class).getMappedResults()
- .stream()
- .map(this::toBillingData)
- .collect(Collectors.toList());
- } catch (Exception e) {
- log.error("Cannot retrieve billing information ", e);
- throw new DlabException("Cannot retrieve billing information", e);
- }
- }
-
- @Override
- public List<BillingData> getBillingReport(List<String> dlabIds) {
- try {
- GroupOperation groupOperation = getGroupOperation();
- MatchOperation matchOperation = Aggregation.match(Criteria.where("dlab_id").in(dlabIds));
- Aggregation aggregation = newAggregation(matchOperation, groupOperation);
-
- return mongoTemplate.aggregate(aggregation, "billing", AzureDailyResourceInvoice.class).getMappedResults()
- .stream()
- .map(this::toBillingData)
- .collect(Collectors.toList());
- } catch (Exception e) {
- log.error("Cannot retrieve billing information ", e);
- throw new DlabException("Cannot retrieve billing information", e);
- }
- }
-
- private GroupOperation getGroupOperation() {
- return group("meterCategory", "currencyCode", "dlabId")
- .min("usageStartDate").as("usageStartDate")
- .max("usageEndDate").as("usageEndDate")
- .sum("cost").as("cost");
- }
-
- private Optional<MatchOperation> getMatchCriteria(String dateStart, Criteria criteria) {
- return Optional.ofNullable(dateStart)
- .filter(StringUtils::isNotEmpty)
- .map(date -> Aggregation.match(criteria));
- }
-
- private BillingData toBillingData(AzureDailyResourceInvoice billingData) {
- return BillingData.builder()
- .tag(billingData.getDlabId())
- .usageDateFrom(Optional.ofNullable(billingData.getUsageStartDate()).map(LocalDate::parse).orElse(null))
- .usageDateTo(Optional.ofNullable(billingData.getUsageEndDate()).map(LocalDate::parse).orElse(null))
- .product(billingData.getMeterCategory())
- .cost(BigDecimal.valueOf(billingData.getCost()).setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue())
- .currency(billingData.getCurrencyCode())
- .build();
- }
-}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingGcpApplication.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingGcpApplication.java
index 0e31323..c454038 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingGcpApplication.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/BillingGcpApplication.java
@@ -23,10 +23,8 @@
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
-import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
-@EnableScheduling
@EnableMongoRepositories
@EnableConfigurationProperties
public class BillingGcpApplication {
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java
index 2967d2a..ea45d89 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/controller/BillingController.java
@@ -19,12 +19,11 @@
package com.epam.dlab.billing.gcp.controller;
-import com.epam.dlab.billing.gcp.dao.BillingDAO;
+import com.epam.dlab.billing.gcp.service.BillingService;
import com.epam.dlab.dto.billing.BillingData;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
@@ -32,22 +31,14 @@
@RestController
public class BillingController {
- private final BillingDAO billingDAO;
+ private final BillingService billingService;
- public BillingController(BillingDAO billingDAO) {
- this.billingDAO = billingDAO;
+ public BillingController(BillingService billingService) {
+ this.billingService = billingService;
}
@GetMapping
- public ResponseEntity<List<BillingData>> getBilling(@RequestParam List<String> dlabIds) {
- return new ResponseEntity<>(billingDAO.getBillingReport(dlabIds), HttpStatus.OK);
- }
-
- @GetMapping("/report")
- public ResponseEntity<List<BillingData>> getBilling(@RequestParam("date-start") String dateStart,
- @RequestParam("date-end") String dateEnd,
- @RequestParam("dlab-id") String dlabId,
- @RequestParam("product") List<String> products) {
- return new ResponseEntity<>(billingDAO.getBillingReport(dateStart, dateEnd, dlabId, products), HttpStatus.OK);
+ public ResponseEntity<List<BillingData>> getBilling() {
+ return new ResponseEntity<>(billingService.getBillingData(), HttpStatus.OK);
}
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
index 430ade7..7c791df 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/BillingDAO.java
@@ -19,16 +19,10 @@
package com.epam.dlab.billing.gcp.dao;
-import com.epam.dlab.billing.gcp.model.GcpBillingData;
import com.epam.dlab.dto.billing.BillingData;
import java.util.List;
public interface BillingDAO {
-
- List<GcpBillingData> getBillingData() throws InterruptedException;
-
- List<BillingData> getBillingReport(String dateStart, String dateEnd, String dlabId, List<String> products);
-
- List<BillingData> getBillingReport(List<String> dlabIds);
+ List<BillingData> getBillingData() throws InterruptedException;
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
index 6db993a..061283d 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/dao/impl/BigQueryBillingDAO.java
@@ -22,10 +22,8 @@
import com.epam.dlab.billing.gcp.conf.DlabConfiguration;
import com.epam.dlab.billing.gcp.dao.BillingDAO;
import com.epam.dlab.billing.gcp.model.BillingHistory;
-import com.epam.dlab.billing.gcp.model.GcpBillingData;
import com.epam.dlab.billing.gcp.repository.BillingHistoryRepository;
import com.epam.dlab.dto.billing.BillingData;
-import com.epam.dlab.exceptions.DlabException;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.FieldValueList;
import com.google.cloud.bigquery.QueryJobConfiguration;
@@ -33,32 +31,20 @@
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableInfo;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.MongoTemplate;
-import org.springframework.data.mongodb.core.aggregation.Aggregation;
-import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
-import org.springframework.data.mongodb.core.aggregation.GroupOperation;
-import org.springframework.data.mongodb.core.aggregation.MatchOperation;
-import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.stereotype.Component;
-import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import static org.springframework.data.mongodb.core.aggregation.Aggregation.group;
-import static org.springframework.data.mongodb.core.aggregation.Aggregation.newAggregation;
-
@Component
@Slf4j
public class BigQueryBillingDAO implements BillingDAO {
@@ -92,7 +78,7 @@
}
@Override
- public List<GcpBillingData> getBillingData() {
+ public List<BillingData> getBillingData() {
final Map<String, Long> processedBillingTables = billingHistoryRepo.findAll()
.stream()
.collect(Collectors.toMap(BillingHistory::getTableName, BillingHistory::getLastModified));
@@ -107,63 +93,7 @@
.collect(Collectors.toList());
}
- @Override
- public List<BillingData> getBillingReport(String dateStart, String dateEnd, String dlabId, List<String> products) {
- try {
- List<AggregationOperation> aggregationOperations = new ArrayList<>();
- aggregationOperations.add(Aggregation.match(Criteria.where("dlabId").regex(dlabId, "i")));
- if (!products.isEmpty()) {
- aggregationOperations.add(Aggregation.match(Criteria.where("product").in(products)));
- }
- getMatchCriteria(dateStart, Criteria.where("usage_date").gte(dateStart))
- .ifPresent(aggregationOperations::add);
- getMatchCriteria(dateEnd, Criteria.where("usage_date").lte(dateEnd))
- .ifPresent(aggregationOperations::add);
- aggregationOperations.add(getGroupOperation());
-
- Aggregation aggregation = newAggregation(aggregationOperations);
-
- return mongoTemplate.aggregate(aggregation, "billing", GcpBillingData.class).getMappedResults()
- .stream()
- .map(this::toBillingData)
- .collect(Collectors.toList());
- } catch (Exception e) {
- log.error("Cannot retrieve billing information ", e);
- throw new DlabException("Cannot retrieve billing information", e);
- }
- }
-
- @Override
- public List<BillingData> getBillingReport(List<String> dlabIds) {
- try {
- GroupOperation groupOperation = getGroupOperation();
- MatchOperation matchOperation = Aggregation.match(Criteria.where("dlabId").in(dlabIds));
- Aggregation aggregation = newAggregation(matchOperation, groupOperation);
-
- return mongoTemplate.aggregate(aggregation, "billing", GcpBillingData.class).getMappedResults()
- .stream()
- .map(this::toBillingData)
- .collect(Collectors.toList());
- } catch (Exception e) {
- log.error("Cannot retrieve billing information ", e);
- throw new DlabException("Cannot retrieve billing information", e);
- }
- }
-
- private GroupOperation getGroupOperation() {
- return group("product", "currency", "dlabId")
- .min("from").as("from")
- .max("to").as("to")
- .sum("cost").as("cost");
- }
-
- private Optional<MatchOperation> getMatchCriteria(String dateStart, Criteria criteria) {
- return Optional.ofNullable(dateStart)
- .filter(StringUtils::isNotEmpty)
- .map(date -> Aggregation.match(criteria));
- }
-
- private Stream<? extends GcpBillingData> bigQueryResultSetStream(Table table) {
+ private Stream<? extends BillingData> bigQueryResultSetStream(Table table) {
try {
final String tableName = table.getTableId().getTable();
final String tableId = table.getTableId().getDataset() + "." + tableName;
@@ -172,7 +102,7 @@
.addNamedParameter(SBN_PARAM, QueryParameterValue.string(sbn + "%"))
.addNamedParameter(DATASET_PARAM, QueryParameterValue.string(tableId))
.build();
- final Stream<GcpBillingData> gcpBillingDataStream =
+ final Stream<BillingData> gcpBillingDataStream =
StreamSupport.stream(service.query(queryConfig).getValues().spliterator(), false)
.map(this::toGcpBillingData);
billingHistoryRepo.save(new BillingHistory(tableName, table.getLastModifiedTime()));
@@ -182,15 +112,15 @@
}
}
- private GcpBillingData toGcpBillingData(FieldValueList fields) {
- return GcpBillingData.builder()
+ private BillingData toGcpBillingData(FieldValueList fields) {
+ return BillingData.builder()
.usageDateFrom(toLocalDate(fields, "usage_date_from"))
.usageDateTo(toLocalDate(fields, "usage_date_to"))
.cost(fields.get("cost").getNumericValue().doubleValue())
.product(fields.get("product").getStringValue())
.usageType(fields.get("usageType").getStringValue())
.currency(fields.get("currency").getStringValue())
- .tag(fields.get("value").getStringValue())
+ .tag(fields.get("value").getStringValue().toLowerCase())
.usageDate(toLocalDate(fields, "usage_date_from").format((DateTimeFormatter.ofPattern(DATE_FORMAT))))
.build();
}
@@ -199,16 +129,4 @@
return LocalDate.from(Instant.ofEpochMilli(fieldValues.get(timestampFieldName).getTimestampValue() / 1000)
.atZone(ZoneId.systemDefault()));
}
-
- private BillingData toBillingData(GcpBillingData billingData) {
- return BillingData.builder()
- .usageDateFrom(billingData.getUsageDateFrom())
- .usageDateTo(billingData.getUsageDateTo())
- .product(billingData.getProduct())
- .usageType(billingData.getUsageType())
- .cost(BigDecimal.valueOf(billingData.getCost()).setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue())
- .currency(billingData.getCurrency())
- .tag(billingData.getTag())
- .build();
- }
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java
deleted file mode 100644
index 9724d43..0000000
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/scheduler/BillingScheduler.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package com.epam.dlab.billing.gcp.scheduler;
-
-import com.epam.dlab.billing.gcp.service.BillingService;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
-
-@Component
-public class BillingScheduler {
-
- private final BillingService billingService;
-
- @Autowired
- public BillingScheduler(BillingService billingService) {
- this.billingService = billingService;
- }
-
-
- @Scheduled(cron = "${dlab.cron}")
- public void getBillingReport() {
- billingService.updateBillingData();
- }
-}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
index bb56d6c..7bb3246 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/BillingService.java
@@ -19,6 +19,10 @@
package com.epam.dlab.billing.gcp.service;
+import com.epam.dlab.dto.billing.BillingData;
+
+import java.util.List;
+
public interface BillingService {
- void updateBillingData();
+ List<BillingData> getBillingData();
}
diff --git a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/impl/BillingServiceImpl.java b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/impl/BillingServiceImpl.java
index c37436b..5661dfb 100644
--- a/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/impl/BillingServiceImpl.java
+++ b/services/billing-gcp/src/main/java/com/epam/dlab/billing/gcp/service/impl/BillingServiceImpl.java
@@ -20,45 +20,33 @@
package com.epam.dlab.billing.gcp.service.impl;
import com.epam.dlab.billing.gcp.dao.BillingDAO;
-import com.epam.dlab.billing.gcp.model.GcpBillingData;
-import com.epam.dlab.billing.gcp.repository.BillingRepository;
import com.epam.dlab.billing.gcp.service.BillingService;
+import com.epam.dlab.dto.billing.BillingData;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import java.util.Collections;
import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
@Service
@Slf4j
public class BillingServiceImpl implements BillingService {
- private static final String USAGE_DATE_FORMAT = "yyyy-MM";
private final BillingDAO billingDAO;
- private final BillingRepository billingRepository;
@Autowired
- public BillingServiceImpl(BillingDAO billingDAO, BillingRepository billingRepository) {
+ public BillingServiceImpl(BillingDAO billingDAO) {
this.billingDAO = billingDAO;
- this.billingRepository = billingRepository;
}
@Override
- public void updateBillingData() {
+ public List<BillingData> getBillingData() {
try {
- Map<String, List<GcpBillingData>> billingData = billingDAO.getBillingData()
- .stream()
- .collect(Collectors.groupingBy(bd -> bd.getUsageDate().substring(0, USAGE_DATE_FORMAT.length())));
-
- billingData.forEach((usageDate, billingDataList) -> {
- log.info("Updating billing information for month {}", usageDate);
- billingRepository.deleteByUsageDateRegex("^" + usageDate);
- billingRepository.insert(billingDataList);
- });
+ return billingDAO.getBillingData();
} catch (Exception e) {
log.error("Can not update billing due to: {}", e.getMessage(), e);
+ return Collections.emptyList();
}
}
}
diff --git a/services/billing-gcp/src/test/java/com/epam/dlab/billing/gcp/service/BillingServiceImplTest.java b/services/billing-gcp/src/test/java/com/epam/dlab/billing/gcp/service/BillingServiceImplTest.java
deleted file mode 100644
index 2ff670a..0000000
--- a/services/billing-gcp/src/test/java/com/epam/dlab/billing/gcp/service/BillingServiceImplTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package com.epam.dlab.billing.gcp.service;
-
-import com.epam.dlab.billing.gcp.dao.BillingDAO;
-import com.epam.dlab.billing.gcp.model.GcpBillingData;
-import com.epam.dlab.billing.gcp.repository.BillingRepository;
-import com.epam.dlab.billing.gcp.service.impl.BillingServiceImpl;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.runners.MockitoJUnitRunner;
-
-import java.time.LocalDate;
-import java.util.Collections;
-import java.util.List;
-
-import static org.mockito.Mockito.anyCollection;
-import static org.mockito.Mockito.anyString;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyNoMoreInteractions;
-import static org.mockito.Mockito.when;
-
-
-@RunWith(MockitoJUnitRunner.class)
-public class BillingServiceImplTest {
- @Mock
- private BillingDAO billingDAO;
- @Mock
- private BillingRepository billingRepository;
- @InjectMocks
- private BillingServiceImpl billingService;
-
- @Test
- public void updateBillingData() throws InterruptedException {
- when(billingDAO.getBillingData()).thenReturn(getBillingData());
-
- billingService.updateBillingData();
-
- verify(billingDAO).getBillingData();
- verify(billingRepository).deleteByUsageDateRegex(anyString());
- verify(billingRepository).insert(anyCollection());
-
- verifyNoMoreInteractions(billingDAO);
- }
-
- private List<GcpBillingData> getBillingData() {
- return Collections.singletonList(GcpBillingData.builder()
- .usageDate(LocalDate.MIN.toString())
- .usageDateFrom(LocalDate.MIN)
- .usageDateTo(LocalDate.MAX)
- .product("product")
- .usageType("usageType")
- .cost(1d)
- .currency("USD")
- .tag("exploratoryId")
- .build());
- }
-}
\ No newline at end of file
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java
index 27db613..c95a02e 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/billing/BillingData.java
@@ -31,6 +31,7 @@
@JsonIgnoreProperties(ignoreUnknown = true)
public class BillingData {
private final String tag;
+ private String application;
@JsonProperty("from")
private LocalDate usageDateFrom;
@JsonProperty("to")
@@ -39,4 +40,5 @@
private String usageType;
private Double cost;
private String currency;
+ private final String usageDate;
}
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java
index 56da2f6..9f8c021 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/computational/UserComputationalResource.java
@@ -59,6 +59,8 @@
private LocalDateTime lastActivity;
@JsonProperty("master_node_shape")
private String masterNodeShape;
+ @JsonProperty("slave_node_shape")
+ private String slaveNodeShape;
@JsonProperty("dataengine_instance_shape")
private String dataengineShape;
@JsonProperty("dataengine_instance_count")
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/exploratory/ExploratoryImageDTO.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/exploratory/ExploratoryImageDTO.java
index 0dad8e4..b41f432 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/dto/exploratory/ExploratoryImageDTO.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/exploratory/ExploratoryImageDTO.java
@@ -35,6 +35,8 @@
private Map<String, String> tags;
@JsonProperty("endpoint_name")
private String endpoint;
+ @JsonProperty("conf_shared_image_enabled")
+ private String sharedImageEnabled;
public ExploratoryImageDTO withImageName(String imageName) {
this.imageName = imageName;
@@ -51,6 +53,11 @@
return this;
}
+ public ExploratoryImageDTO withSharedImageEnabled(String sharedImageEnabled) {
+ this.sharedImageEnabled = sharedImageEnabled;
+ return this;
+ }
+
@Override
public MoreObjects.ToStringHelper toStringHelper(Object self) {
return super.toStringHelper(self)
diff --git a/services/dlab-webapp-common/src/main/java/com/epam/dlab/ServiceConfiguration.java b/services/dlab-webapp-common/src/main/java/com/epam/dlab/ServiceConfiguration.java
index 85d4318..3297abd 100644
--- a/services/dlab-webapp-common/src/main/java/com/epam/dlab/ServiceConfiguration.java
+++ b/services/dlab-webapp-common/src/main/java/com/epam/dlab/ServiceConfiguration.java
@@ -54,6 +54,11 @@
@Valid
@NotNull
+ @JsonProperty(ServiceConsts.BILLING_SERVICE_NAME)
+ private RESTServiceFactory billingFactory = new RESTServiceFactory();
+
+ @Valid
+ @NotNull
@JsonProperty(ServiceConsts.SECURITY_SERVICE_NAME)
private RESTServiceFactory securityFactory;
@@ -85,6 +90,10 @@
return provisioningFactory;
}
+ public RESTServiceFactory getBillingFactory() {
+ return billingFactory;
+ }
+
public RESTServiceFactory getSecurityFactory() {
return securityFactory;
}
diff --git a/services/dlab-webapp-common/src/main/java/com/epam/dlab/constants/ServiceConsts.java b/services/dlab-webapp-common/src/main/java/com/epam/dlab/constants/ServiceConsts.java
index d376665..e1bcf23 100644
--- a/services/dlab-webapp-common/src/main/java/com/epam/dlab/constants/ServiceConsts.java
+++ b/services/dlab-webapp-common/src/main/java/com/epam/dlab/constants/ServiceConsts.java
@@ -20,13 +20,14 @@
package com.epam.dlab.constants;
public final class ServiceConsts {
- public static final String MONGO_NAME = "mongo";
- public static final String PROVISIONING_SERVICE_NAME = "provisioningService";
- public static final String MAVEN_SEARCH_API = "mavenSearchService";
- public static final String SECURITY_SERVICE_NAME = "securityService";
- public static final String SELF_SERVICE_NAME = "selfService";
- public static final String PROVISIONING_USER_AGENT = "provisioning-service";
+ public static final String MONGO_NAME = "mongo";
+ public static final String PROVISIONING_SERVICE_NAME = "provisioningService";
+ public static final String BILLING_SERVICE_NAME = "billingService";
+ public static final String MAVEN_SEARCH_API = "mavenSearchService";
+ public static final String SECURITY_SERVICE_NAME = "securityService";
+ public static final String SELF_SERVICE_NAME = "selfService";
+ public static final String PROVISIONING_USER_AGENT = "provisioning-service";
- private ServiceConsts() {
- }
+ private ServiceConsts() {
+ }
}
diff --git a/services/self-service/self-service.yml b/services/self-service/self-service.yml
index 53bd131..df92c25 100644
--- a/services/self-service/self-service.yml
+++ b/services/self-service/self-service.yml
@@ -140,16 +140,19 @@
cron: "*/20 * * ? * * *"
checkQuoteScheduler:
enabled: true
- cron: "0 0 * ? * * *"
+ cron: "0 2/15 * ? * *"
checkUserQuoteScheduler:
enabled: false
cron: "0 0 * ? * * *"
checkProjectQuoteScheduler:
enabled: true
- cron: "0 * * ? * * *"
+ cron: "0 4/15 * ? * *"
checkEndpointStatusScheduler:
enabled: true
- cron: "0 */15 * ? * *"
+ cron: "0 6/15 * ? * *"
+ billingScheduler:
+ enabled: true
+ cron: "0 0/15 * ? * *"
guacamole:
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
index d22e400..28a6c64 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseBillingDAO.java
@@ -19,30 +19,62 @@
package com.epam.dlab.backendapi.dao;
+import com.epam.dlab.backendapi.domain.BillingReportLine;
+import com.epam.dlab.backendapi.resources.dto.BillingFilter;
+import com.epam.dlab.dto.billing.BillingResourceType;
import com.google.inject.Inject;
+import com.mongodb.client.model.Aggregates;
+import com.mongodb.client.model.Filters;
import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.bson.Document;
import org.bson.conversions.Bson;
import java.math.BigDecimal;
+import java.time.ZoneId;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import static com.epam.dlab.backendapi.dao.MongoCollections.BILLING;
+import static com.mongodb.client.model.Accumulators.max;
+import static com.mongodb.client.model.Accumulators.min;
import static com.mongodb.client.model.Accumulators.sum;
import static com.mongodb.client.model.Aggregates.group;
import static com.mongodb.client.model.Aggregates.match;
+import static com.mongodb.client.model.Filters.and;
import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.gte;
+import static com.mongodb.client.model.Filters.in;
+import static com.mongodb.client.model.Filters.lte;
+import static com.mongodb.client.model.Filters.regex;
import static java.util.Collections.singletonList;
@Slf4j
public class BaseBillingDAO extends BaseDAO implements BillingDAO {
-
- private static final String PROJECT = "project";
private static final int ONE_HUNDRED = 100;
- private static final String TOTAL_FIELD_NAME = "total";
private static final String COST_FIELD = "$cost";
+ private static final String TOTAL_FIELD_NAME = "total";
+ private static final String PROJECT = "project";
+ private static final String APPLICATION = "application";
+ private static final String USAGE_DATE = "usageDate";
+ private static final String USER = "user";
+ private static final String RESOURCE_TYPE = "resource_type";
+ private static final String DLAB_ID = "dlabId";
+ private static final String FROM = "from";
+ private static final String TO = "to";
+ private static final String PRODUCT = "product";
+ private static final String CURRENCY = "currency";
+ private static final String COST = "cost";
+ private static final String RESOURCE_NAME = "resource_name";
+ private static final String ENDPOINT = "endpoint";
+ private static final String SHAPE = "shape";
+ private static final String EXPLORATORY = "exploratoryName";
@Inject
protected SettingsDAO settings;
@@ -102,10 +134,44 @@
}
@Override
+ public List<BillingReportLine> findBillingData(String project, String endpoint, List<String> resourceNames) {
+ return find(BILLING, and(eq(PROJECT, project), eq(ENDPOINT, endpoint), in(RESOURCE_NAME, resourceNames)), BillingReportLine.class);
+ }
+
+ @Override
public int getBillingProjectQuoteUsed(String project) {
return toPercentage(() -> projectDAO.getAllowedBudget(project), getProjectCost(project));
}
+ public List<BillingReportLine> aggregateBillingData(BillingFilter filter) {
+ List<Bson> pipeline = new ArrayList<>();
+ List<Bson> matchCriteria = matchCriteria(filter);
+ if (!matchCriteria.isEmpty()) {
+ pipeline.add(Aggregates.match(Filters.and(matchCriteria)));
+ }
+ pipeline.add(groupCriteria());
+ return StreamSupport.stream(getCollection(BILLING).aggregate(pipeline).spliterator(), false)
+ .map(this::toBillingReport)
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public void deleteByUsageDate(String application, String usageDate) {
+ deleteMany(BILLING, and(eq(APPLICATION, application), eq(USAGE_DATE, usageDate)));
+ }
+
+ @Override
+ public void deleteByUsageDateRegex(String application, String usageDate) {
+ deleteMany(BILLING, and(eq(APPLICATION, application), regex(USAGE_DATE, "^" + usageDate)));
+ }
+
+ @Override
+ public void save(List<BillingReportLine> billingData) {
+ if (CollectionUtils.isNotEmpty(billingData)) {
+ insertMany(BILLING, new ArrayList<>(billingData));
+ }
+ }
+
private Integer toPercentage(Supplier<Optional<Integer>> allowedBudget, Double totalCost) {
return allowedBudget.get()
.map(userBudget -> (totalCost * ONE_HUNDRED) / userBudget)
@@ -118,4 +184,60 @@
.map(d -> d.getDouble(TOTAL_FIELD_NAME))
.orElse(BigDecimal.ZERO.doubleValue());
}
+
+ private Bson groupCriteria() {
+ return group(getGroupingFields(USER, DLAB_ID, RESOURCE_TYPE, RESOURCE_NAME, PROJECT, PRODUCT, CURRENCY, SHAPE, EXPLORATORY),
+ sum(COST, "$" + COST),
+ min(FROM, "$" + FROM),
+ max(TO, "$" + TO));
+ }
+
+ private List<Bson> matchCriteria(BillingFilter filter) {
+ List<Bson> searchCriteria = new ArrayList<>();
+
+ if (CollectionUtils.isNotEmpty(filter.getUsers())) {
+ searchCriteria.add(in(USER, filter.getUsers()));
+ }
+ if (CollectionUtils.isNotEmpty(filter.getResourceTypes())) {
+ searchCriteria.add(in(RESOURCE_TYPE, filter.getResourceTypes()));
+ }
+ if (StringUtils.isNotEmpty(filter.getDlabId())) {
+ searchCriteria.add(regex(DLAB_ID, filter.getDlabId(), "i"));
+ }
+ if (StringUtils.isNotEmpty(filter.getDateStart())) {
+ searchCriteria.add(gte(USAGE_DATE, filter.getDateStart()));
+ }
+ if (StringUtils.isNotEmpty(filter.getDateEnd())) {
+ searchCriteria.add(lte(USAGE_DATE, filter.getDateEnd()));
+ }
+ if (CollectionUtils.isNotEmpty(filter.getProjects())) {
+ searchCriteria.add(in(PROJECT, filter.getProjects()));
+ }
+ if (CollectionUtils.isNotEmpty(filter.getProducts())) {
+ searchCriteria.add(in(PRODUCT, filter.getProducts()));
+ }
+ if (CollectionUtils.isNotEmpty(filter.getShapes())) {
+ searchCriteria.add(regex(SHAPE, "(" + String.join("|", filter.getShapes()) + ")"));
+ }
+
+ return searchCriteria;
+ }
+
+ private BillingReportLine toBillingReport(Document d) {
+ Document id = (Document) d.get("_id");
+ return BillingReportLine.builder()
+ .dlabId(id.getString(DLAB_ID))
+ .project(id.getString(PROJECT))
+ .resourceName(id.getString(RESOURCE_NAME))
+ .exploratoryName(id.getString(EXPLORATORY))
+ .shape(id.getString(SHAPE))
+ .user(id.getString(USER))
+ .product(id.getString(PRODUCT))
+ .resourceType(Optional.ofNullable(id.getString(RESOURCE_TYPE)).map(BillingResourceType::valueOf).orElse(null))
+ .usageDateFrom(d.getDate(FROM).toInstant().atZone(ZoneId.systemDefault()).toLocalDate())
+ .usageDateTo(d.getDate(TO).toInstant().atZone(ZoneId.systemDefault()).toLocalDate())
+ .cost(BigDecimal.valueOf(d.getDouble(COST)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue())
+ .currency(id.getString(CURRENCY))
+ .build();
+ }
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseDAO.java
index 034011a..c2ff69b 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BaseDAO.java
@@ -31,7 +31,11 @@
import com.google.inject.Inject;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoException;
-import com.mongodb.client.*;
+import com.mongodb.client.AggregateIterable;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.MongoIterable;
import com.mongodb.client.model.UpdateOptions;
import com.mongodb.client.result.DeleteResult;
import com.mongodb.client.result.UpdateResult;
@@ -41,13 +45,21 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import static com.mongodb.client.model.Filters.*;
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.exists;
+import static com.mongodb.client.model.Filters.ne;
/**
* Implements the base API for Mongo database.
@@ -158,6 +170,29 @@
}
/**
+ * Serializes objects and inserts into the collection.
+ *
+ * @param collection collection name.
+ * @param object for inserting to collection.
+ */
+ protected void insertMany(String collection, List<Object> object) {
+ try {
+ mongoService.getCollection(collection)
+ .insertMany(convertToBson(object)
+ .stream()
+ .peek(o -> {
+ o.append(ID, generateUUID());
+ o.append(TIMESTAMP, new Date());
+ })
+ .collect(Collectors.toList())
+ );
+ } catch (MongoException e) {
+ LOGGER.warn("Insert to Mongo DB fails: {}", e.getLocalizedMessage(), e);
+ throw new DlabException("Insert to Mongo DB fails: " + e.getLocalizedMessage(), e);
+ }
+ }
+
+ /**
* Updates single document in the collection by condition.
*
* @param collection collection name.
@@ -230,6 +265,22 @@
}
/**
+ * Removes many documents in the collection by condition.
+ *
+ * @param collection collection name.
+ * @param condition condition for search documents in collection.
+ */
+ protected DeleteResult deleteMany(String collection, Bson condition) {
+ try {
+ return mongoService.getCollection(collection)
+ .deleteMany(condition);
+ } catch (MongoException e) {
+ LOGGER.warn("Removing document from Mongo DB fails: {}", e.getLocalizedMessage(), e);
+ throw new DlabException("Removing document from Mongo DB fails: " + e.getLocalizedMessage(), e);
+ }
+ }
+
+ /**
* Finds and returns all documents from the collection.
*
* @param collection collection name.
@@ -362,6 +413,13 @@
}
}
+ List<Document> convertToBson(List<Object> objects) {
+ return objects
+ .stream()
+ .map(this::convertToBson)
+ .collect(Collectors.toList());
+ }
+
/**
* Finds and returns one object as given class from the collection by condition.
*
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BillingDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BillingDAO.java
index d50c62f..67630cd 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BillingDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/BillingDAO.java
@@ -18,6 +18,11 @@
*/
package com.epam.dlab.backendapi.dao;
+import com.epam.dlab.backendapi.domain.BillingReportLine;
+import com.epam.dlab.backendapi.resources.dto.BillingFilter;
+
+import java.util.List;
+
public interface BillingDAO {
Double getTotalCost();
@@ -36,4 +41,14 @@
boolean isUserQuoteReached(String user);
boolean isProjectQuoteReached(String project);
+
+ List<BillingReportLine> findBillingData(String project, String endpoint, List<String> resourceNames);
+
+ List<BillingReportLine> aggregateBillingData(BillingFilter filter);
+
+ void deleteByUsageDate(String application, String usageDate);
+
+ void deleteByUsageDateRegex(String application, String usageDate);
+
+ void save(List<BillingReportLine> billingData);
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDao.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDao.java
index c9b5585..48abb54 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDao.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDao.java
@@ -47,5 +47,5 @@
boolean removeGroup(String groupId);
- List<UserGroupDto> aggregateRolesByGroup(boolean isAdmin);
+ List<UserGroupDto> aggregateRolesByGroup();
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDaoImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDaoImpl.java
index c256791..5bc845a 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDaoImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/UserRoleDaoImpl.java
@@ -21,12 +21,11 @@
import com.epam.dlab.backendapi.resources.dto.UserGroupDto;
import com.epam.dlab.backendapi.resources.dto.UserRoleDto;
import com.epam.dlab.cloud.CloudProvider;
+import com.epam.dlab.exceptions.DlabException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Singleton;
-import com.mongodb.client.model.Aggregates;
import com.mongodb.client.model.BsonField;
-import com.mongodb.client.model.Filters;
import com.mongodb.client.result.UpdateResult;
import org.bson.Document;
import org.bson.conversions.Bson;
@@ -35,9 +34,12 @@
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static com.epam.dlab.backendapi.dao.MongoCollections.USER_GROUPS;
import static com.mongodb.client.model.Aggregates.group;
@@ -67,7 +69,6 @@
private static final String EXPLORATORIES_FIELD = "exploratories";
private static final String COMPUTATIONALS_FIELD = "computationals";
private static final String GROUP_INFO = "groupInfo";
- private static final String ADMIN = "admin";
@Override
@@ -95,11 +96,20 @@
@Override
public void updateMissingRoles(CloudProvider cloudProvider) {
- getUserRoleFromFile(cloudProvider).stream()
- .filter(u -> findAll().stream()
+ getUserRoleFromFile(cloudProvider)
+ .stream()
+ .peek(u -> u.setGroups(Collections.emptySet()))
+ .filter(u -> findAll()
+ .stream()
.map(UserRoleDto::getId)
.noneMatch(id -> id.equals(u.getId())))
.forEach(this::insert);
+
+ addGroupToRole(aggregateRolesByGroup()
+ .stream()
+ .map(UserGroupDto::getGroup)
+ .collect(Collectors.toSet()),
+ getDefaultShapes(cloudProvider));
}
@Override
@@ -142,19 +152,15 @@
}
@Override
- public List<UserGroupDto> aggregateRolesByGroup(boolean isAdmin) {
+ public List<UserGroupDto> aggregateRolesByGroup() {
final Document role = roleDocument();
final Bson groupBy = group(GROUPS, new BsonField(ROLES, new Document(ADD_TO_SET, role)));
final Bson lookup = lookup(USER_GROUPS, ID, ID, GROUP_INFO);
- final List<Bson> pipeline = new ArrayList<>();
- if (!isAdmin) {
- pipeline.add(Aggregates.match(Filters.not(eq(ID, ADMIN))));
- }
- pipeline.addAll(Arrays.asList(unwind(GROUPS), groupBy, lookup,
+ final List<Bson> pipeline = Arrays.asList(unwind(GROUPS), groupBy, lookup,
project(new Document(GROUP, "$" + ID).append(GROUP_INFO, elementAt(GROUP_INFO, 0))
.append(ROLES, "$" + ROLES)),
project(new Document(GROUP, "$" + ID).append(USERS_FIELD, "$" + GROUP_INFO + "." + USERS_FIELD)
- .append(ROLES, "$" + ROLES))));
+ .append(ROLES, "$" + ROLES)));
return stream(aggregate(MongoCollections.ROLES, pipeline))
.map(d -> convertFromDocument(d, UserGroupDto.class))
@@ -170,6 +176,21 @@
}
}
+ private Set<String> getDefaultShapes(CloudProvider cloudProvider) {
+ if (cloudProvider == CloudProvider.AWS) {
+ return Stream.of("nbShapes_t2.medium_fetching", "compShapes_c4.xlarge_fetching")
+ .collect(Collectors.toSet());
+ } else if (cloudProvider == CloudProvider.GCP) {
+ return Stream.of("compShapes_n1-standard-2_fetching", "nbShapes_n1-standard-2_fetching")
+ .collect(Collectors.toSet());
+ } else if (cloudProvider == CloudProvider.AZURE) {
+ return Stream.of("nbShapes_Standard_E4s_v3_fetching", "compShapes_Standard_E4s_v3_fetching")
+ .collect(Collectors.toSet());
+ } else {
+ throw new DlabException("Unsupported cloud provider " + cloudProvider);
+ }
+ }
+
private Document roleDocument() {
return new Document().append(ID, "$" + ID)
.append(DESCRIPTION, "$" + DESCRIPTION)
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReport.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReport.java
index 8722e73..2bb2062 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReport.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReport.java
@@ -30,6 +30,7 @@
@Builder
public class BillingReport {
private String sbn;
+ private String name;
@JsonProperty("report_lines")
private List<BillingReportLine> reportLines;
@JsonProperty("from")
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportLine.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportLine.java
index ed97a44..a9cdd12 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportLine.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/BillingReportLine.java
@@ -21,6 +21,7 @@
import com.epam.dlab.dto.UserInstanceStatus;
import com.epam.dlab.dto.billing.BillingResourceType;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
@@ -29,16 +30,20 @@
@Data
@Builder
+@JsonIgnoreProperties(ignoreUnknown = true)
public class BillingReportLine {
private String dlabId;
+ private String application;
@JsonProperty("resource_name")
private String resourceName;
private String project;
+ private String endpoint;
private String user;
@JsonProperty("from")
private LocalDate usageDateFrom;
@JsonProperty("to")
private LocalDate usageDateTo;
+ private String usageDate;
private String product;
private String usageType;
private Double cost;
@@ -47,4 +52,5 @@
private BillingResourceType resourceType;
private UserInstanceStatus status;
private String shape;
+ private String exploratoryName;
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/DevModule.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/DevModule.java
index 3c00111..9275319 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/DevModule.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/DevModule.java
@@ -132,6 +132,9 @@
bind(RESTService.class).annotatedWith(Names.named(ServiceConsts.PROVISIONING_SERVICE_NAME))
.toInstance(configuration.getProvisioningFactory()
.build(environment, ServiceConsts.PROVISIONING_SERVICE_NAME));
+ bind(RESTService.class).annotatedWith(Names.named(ServiceConsts.BILLING_SERVICE_NAME))
+ .toInstance(configuration.getBillingFactory()
+ .build(environment, ServiceConsts.BILLING_SERVICE_NAME));
bind(ImageExploratoryService.class).to(ImageExploratoryServiceImpl.class);
bind(ImageExploratoryDao.class).to(ImageExploratoryDaoImpl.class);
bind(BackupService.class).to(BackupServiceImpl.class);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
index ce7ac26..d20adbf 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
@@ -123,6 +123,9 @@
bind(RESTService.class).annotatedWith(Names.named(ServiceConsts.PROVISIONING_SERVICE_NAME))
.toInstance(configuration.getProvisioningFactory().build(environment, ServiceConsts
.PROVISIONING_SERVICE_NAME));
+ bind(RESTService.class).annotatedWith(Names.named(ServiceConsts.BILLING_SERVICE_NAME))
+ .toInstance(configuration.getBillingFactory()
+ .build(environment, ServiceConsts.BILLING_SERVICE_NAME));
bind(ImageExploratoryService.class).to(ImageExploratoryServiceImpl.class);
bind(ImageExploratoryDao.class).to(ImageExploratoryDaoImpl.class);
bind(BackupService.class).to(BackupServiceImpl.class);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/UserRoleResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/UserRoleResource.java
index b74ef1c..52ad739 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/UserRoleResource.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/UserRoleResource.java
@@ -51,7 +51,7 @@
@GET
public Response getRoles(@Auth UserInfo userInfo) {
log.debug("Getting all roles for admin {}...", userInfo.getName());
- return Response.ok(userRoleService.getUserRoles(userInfo)).build();
+ return Response.ok(userRoleService.getUserRoles()).build();
}
@POST
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ImageInfoRecord.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ImageInfoRecord.java
index d430701..ed722ee 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ImageInfoRecord.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ImageInfoRecord.java
@@ -30,6 +30,7 @@
private final String description;
private final String project;
private final String endpoint;
+ private final String user;
private final String application;
private final String fullName;
private final ImageStatus status;
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
index e1f09e9..b9dfd89 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
@@ -19,8 +19,8 @@
package com.epam.dlab.backendapi.resources.dto;
+import com.epam.dlab.backendapi.domain.BillingReport;
import com.epam.dlab.backendapi.domain.EndpointDTO;
-import com.epam.dlab.dto.billing.BillingData;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.ToString;
@@ -41,7 +41,7 @@
@JsonProperty
private Iterable<Document> exploratory;
@JsonProperty
- private List<BillingData> exploratoryBilling;
+ private List<BillingReport> exploratoryBilling;
@JsonProperty
private List<EndpointDTO> endpoints;
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/schedulers/billing/BillingScheduler.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/schedulers/billing/BillingScheduler.java
new file mode 100644
index 0000000..45563a2
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/schedulers/billing/BillingScheduler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.schedulers.billing;
+
+import com.epam.dlab.backendapi.schedulers.internal.Scheduled;
+import com.epam.dlab.backendapi.service.BillingService;
+import com.epam.dlab.backendapi.service.SecurityService;
+import com.google.inject.Inject;
+import lombok.extern.slf4j.Slf4j;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+
+@Scheduled("billingScheduler")
+@Slf4j
+public class BillingScheduler implements Job {
+
+ private final BillingService billingService;
+ private final SecurityService securityService;
+
+ @Inject
+ public BillingScheduler(BillingService billingService, SecurityService securityService) {
+ this.billingService = billingService;
+ this.securityService = securityService;
+ }
+
+ @Override
+ public void execute(JobExecutionContext jobExecutionContext) {
+ log.info("Trying to update billing");
+ try {
+ billingService.updateRemoteBillingData(securityService.getServiceAccountInfo("admin"));
+ } catch (Exception e) {
+ log.error("Something went wrong {}", e.getMessage());
+ }
+ }
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
index 40a5c14..b76b141 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/BillingService.java
@@ -21,10 +21,7 @@
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.domain.BillingReport;
-import com.epam.dlab.backendapi.domain.BillingReportLine;
import com.epam.dlab.backendapi.resources.dto.BillingFilter;
-import com.epam.dlab.dto.UserInstanceDTO;
-import com.epam.dlab.dto.billing.BillingData;
import java.util.List;
@@ -33,7 +30,7 @@
String downloadReport(UserInfo userInfo, BillingFilter filter);
- List<BillingReportLine> getBillingReportLines(UserInfo userInfo, BillingFilter filter);
+ BillingReport getExploratoryBillingData(String project, String endpoint, String exploratoryName, List<String> compNames);
- List<BillingData> getExploratoryRemoteBillingData(UserInfo user, String endpoint, List<UserInstanceDTO> userInstanceDTOS);
+ void updateRemoteBillingData(UserInfo userInfo);
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
index 5a43fa6..807df17 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/ExploratoryService.java
@@ -48,6 +48,8 @@
Optional<UserInstanceDTO> getUserInstance(String user, String project, String exploratoryName);
+ Optional<UserInstanceDTO> getUserInstance(String user, String project, String exploratoryName, boolean includeCompResources);
+
List<UserInstanceDTO> findAll();
List<UserInstanceDTO> findAll(Set<ProjectDTO> projects);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/UserRoleService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/UserRoleService.java
index a010684..0b22b1d 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/UserRoleService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/UserRoleService.java
@@ -18,14 +18,13 @@
*/
package com.epam.dlab.backendapi.service;
-import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.resources.dto.UserRoleDto;
import java.util.List;
public interface UserRoleService {
- List<UserRoleDto> getUserRoles(UserInfo userInfo);
+ List<UserRoleDto> getUserRoles();
void createRole(UserRoleDto dto);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/UserRoleServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/UserRoleServiceImpl.java
index 6940533..92e0afb 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/UserRoleServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/UserRoleServiceImpl.java
@@ -18,11 +18,8 @@
*/
package com.epam.dlab.backendapi.service;
-import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.dao.UserRoleDao;
import com.epam.dlab.backendapi.resources.dto.UserRoleDto;
-import com.epam.dlab.backendapi.roles.UserRoles;
-import com.epam.dlab.exceptions.DlabException;
import com.epam.dlab.exceptions.ResourceNotFoundException;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -30,29 +27,17 @@
import java.util.Collections;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
@Singleton
public class UserRoleServiceImpl implements UserRoleService {
private static final String ROLE_NOT_FOUND_MSG = "Any of role : %s were not found";
- private static final String ADMIN = "admin";
@Inject
private UserRoleDao userRoleDao;
@Override
- public List<UserRoleDto> getUserRoles(UserInfo user) {
- List<UserRoleDto> all = userRoleDao.findAll();
- if (UserRoles.isAdmin(user)) {
- return all;
- } else if (UserRoles.isProjectAdmin(user)) {
- return all
- .stream()
- .filter(role -> !role.getId().equalsIgnoreCase(ADMIN))
- .collect(Collectors.toList());
- } else {
- throw new DlabException(String.format("User %s doesn't have appropriate permission", user));
- }
+ public List<UserRoleDto> getUserRoles() {
+ return userRoleDao.findAll();
}
@Override
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
index 176f22b..8eae49b 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/BillingServiceImpl.java
@@ -21,6 +21,7 @@
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.conf.SelfServiceApplicationConfiguration;
+import com.epam.dlab.backendapi.dao.BillingDAO;
import com.epam.dlab.backendapi.dao.ImageExploratoryDao;
import com.epam.dlab.backendapi.domain.BillingReport;
import com.epam.dlab.backendapi.domain.BillingReportLine;
@@ -35,9 +36,11 @@
import com.epam.dlab.backendapi.service.ExploratoryService;
import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.backendapi.util.BillingUtils;
+import com.epam.dlab.cloud.CloudProvider;
import com.epam.dlab.constants.ServiceConsts;
-import com.epam.dlab.dto.UserInstanceDTO;
+import com.epam.dlab.dto.UserInstanceStatus;
import com.epam.dlab.dto.billing.BillingData;
+import com.epam.dlab.dto.billing.BillingResourceType;
import com.epam.dlab.exceptions.DlabException;
import com.epam.dlab.rest.client.RESTService;
import com.google.common.collect.Lists;
@@ -61,19 +64,13 @@
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
public class BillingServiceImpl implements BillingService {
private static final String BILLING_PATH = "/api/billing";
- private static final String BILLING_REPORT_PATH = "/api/billing/report";
+ private static final String USAGE_DATE_FORMAT = "yyyy-MM";
private final ProjectService projectService;
private final EndpointService endpointService;
@@ -81,34 +78,43 @@
private final SelfServiceApplicationConfiguration configuration;
private final RESTService provisioningService;
private final ImageExploratoryDao imageExploratoryDao;
+ private final BillingDAO billingDAO;
private final String sbn;
@Inject
public BillingServiceImpl(ProjectService projectService, EndpointService endpointService,
ExploratoryService exploratoryService, SelfServiceApplicationConfiguration configuration,
- @Named(ServiceConsts.PROVISIONING_SERVICE_NAME) RESTService provisioningService, ImageExploratoryDao imageExploratoryDao) {
+ @Named(ServiceConsts.BILLING_SERVICE_NAME) RESTService provisioningService, ImageExploratoryDao imageExploratoryDao,
+ BillingDAO billingDAO) {
this.projectService = projectService;
this.endpointService = endpointService;
this.exploratoryService = exploratoryService;
this.configuration = configuration;
this.provisioningService = provisioningService;
this.imageExploratoryDao = imageExploratoryDao;
+ this.billingDAO = billingDAO;
sbn = configuration.getServiceBaseName();
}
@Override
public BillingReport getBillingReport(UserInfo user, BillingFilter filter) {
- List<BillingReportLine> billingReportLines = getBillingReportLines(user, filter);
- LocalDate min = billingReportLines.stream().min(Comparator.comparing(BillingReportLine::getUsageDateFrom)).map(BillingReportLine::getUsageDateFrom).orElse(null);
- LocalDate max = billingReportLines.stream().max(Comparator.comparing(BillingReportLine::getUsageDateTo)).map(BillingReportLine::getUsageDateTo).orElse(null);
- double sum = billingReportLines.stream().mapToDouble(BillingReportLine::getCost).sum();
- String currency = billingReportLines.stream().map(BillingReportLine::getCurrency).distinct().count() == 1 ? billingReportLines.get(0).getCurrency() : null;
+ setUserFilter(user, filter);
+ List<BillingReportLine> billingReportLines = billingDAO.aggregateBillingData(filter)
+ .stream()
+ .peek(this::appendStatuses)
+ .filter(bd -> CollectionUtils.isEmpty(filter.getStatuses()) || filter.getStatuses().contains(bd.getStatus()))
+ .collect(Collectors.toList());
+ final LocalDate min = billingReportLines.stream().min(Comparator.comparing(BillingReportLine::getUsageDateFrom)).map(BillingReportLine::getUsageDateFrom).orElse(null);
+ final LocalDate max = billingReportLines.stream().max(Comparator.comparing(BillingReportLine::getUsageDateTo)).map(BillingReportLine::getUsageDateTo).orElse(null);
+ final double sum = billingReportLines.stream().mapToDouble(BillingReportLine::getCost).sum();
+ final String currency = billingReportLines.stream().map(BillingReportLine::getCurrency).distinct().count() == 1 ? billingReportLines.get(0).getCurrency() : null;
return BillingReport.builder()
+ .name("Billing report")
.sbn(sbn)
.reportLines(billingReportLines)
.usageDateFrom(min)
.usageDateTo(max)
- .totalCost(new BigDecimal(sum).setScale(3, BigDecimal.ROUND_HALF_UP).doubleValue())
+ .totalCost(new BigDecimal(sum).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue())
.currency(currency)
.isFull(isFullReport(user))
.build();
@@ -116,11 +122,12 @@
@Override
public String downloadReport(UserInfo user, BillingFilter filter) {
+ boolean isFull = isFullReport(user);
BillingReport report = getBillingReport(user, filter);
StringBuilder builder = new StringBuilder(BillingUtils.getFirstLine(report.getSbn(), report.getUsageDateFrom(), report.getUsageDateTo()));
- builder.append(BillingUtils.getHeader());
+ builder.append(BillingUtils.getHeader(isFull));
try {
- report.getReportLines().forEach(r -> builder.append(BillingUtils.printLine(r)));
+ report.getReportLines().forEach(r -> builder.append(BillingUtils.printLine(r, isFull)));
builder.append(BillingUtils.getTotal(report.getTotalCost(), report.getCurrency()));
return builder.toString();
} catch (Exception e) {
@@ -129,54 +136,67 @@
}
}
- @Override
- public List<BillingReportLine> getBillingReportLines(UserInfo user, BillingFilter filter) {
- setUserFilter(user, filter);
- Set<ProjectDTO> projects = new HashSet<>(projectService.getProjects(user));
- projects.addAll(projectService.getUserProjects(user, false));
-
- final Map<String, BillingReportLine> billableResources = getBillableResources(user, projects);
-
- List<BillingReportLine> billingReport = getRemoteBillingData(user, filter)
+ public BillingReport getExploratoryBillingData(String project, String endpoint, String exploratoryName, List<String> compNames) {
+ List<String> resourceNames = new ArrayList<>(compNames);
+ resourceNames.add(exploratoryName);
+ List<BillingReportLine> billingData = billingDAO.findBillingData(project, endpoint, resourceNames)
.stream()
- .filter(bd -> billableResources.containsKey(bd.getTag()))
- .map(bd -> toBillingData(bd, billableResources.get(bd.getTag())))
- .filter(getBillingReportFilter(filter))
+ .peek(bd -> bd.setCost(BigDecimal.valueOf(bd.getCost()).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()))
.collect(Collectors.toList());
- log.debug("Billing report: {}", billingReport);
-
- return billingReport;
+ final double sum = billingData.stream().mapToDouble(BillingReportLine::getCost).sum();
+ final String currency = billingData.stream().map(BillingReportLine::getCurrency).distinct().count() == 1 ? billingData.get(0).getCurrency() : null;
+ return BillingReport.builder()
+ .name(exploratoryName)
+ .reportLines(billingData)
+ .totalCost(new BigDecimal(sum).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue())
+ .currency(currency)
+ .build();
}
- private Map<String, BillingReportLine> getBillableResources(UserInfo user, Set<ProjectDTO> projects) {
- Stream<BillingReportLine> billableAdminResources = Stream.empty();
+ public void updateRemoteBillingData(UserInfo userInfo) {
+ List<EndpointDTO> endpoints = endpointService.getEndpoints();
+ if (CollectionUtils.isEmpty(endpoints)) {
+ log.error("Cannot update billing info. There are no endpoints");
+ throw new DlabException("Cannot update billing info. There are no endpoints");
+ }
+
+ Map<EndpointDTO, List<BillingData>> billingDataMap = endpoints
+ .stream()
+ .collect(Collectors.toMap(e -> e, e -> getBillingData(userInfo, e)));
+
+ billingDataMap.forEach((endpointDTO, billingData) -> {
+ log.info("Updating billing information for endpoint {}. Billing data {}", endpointDTO.getName(), billingData);
+ try {
+ updateBillingData(endpointDTO, billingData);
+ } catch (Exception e) {
+ log.error("Something went wrong while trying to update billing for {}. {}", endpointDTO.getName(), e.getMessage());
+ }
+ });
+ }
+
+ private Map<String, BillingReportLine> getBillableResources() {
+ Set<ProjectDTO> projects = new HashSet<>(projectService.getProjects());
+ final Stream<BillingReportLine> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(sbn);
+ final Stream<BillingReportLine> billableEdges = projects
+ .stream()
+ .collect(Collectors.toMap(ProjectDTO::getName, ProjectDTO::getEndpoints))
+ .entrySet()
+ .stream()
+ .flatMap(e -> projectEdges(sbn, e.getKey(), e.getValue()));
+ final Stream<BillingReportLine> billableSharedEndpoints = endpointService.getEndpoints()
+ .stream()
+ .flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), sbn));
final Stream<BillingReportLine> billableUserInstances = exploratoryService.findAll(projects)
.stream()
.filter(userInstance -> Objects.nonNull(userInstance.getExploratoryId()))
- .flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount(), sbn));
- final Stream<BillingReportLine> billingReportLineStream = projects
+ .flatMap(ui -> BillingUtils.exploratoryBillingDataStream(ui, configuration.getMaxSparkInstanceCount()));
+ final Stream<BillingReportLine> customImages = projects
.stream()
.map(p -> imageExploratoryDao.getImagesForProject(p.getName()))
.flatMap(Collection::stream)
.flatMap(i -> BillingUtils.customImageBillingDataStream(i, sbn));
- if (UserRoles.isAdmin(user)) {
- final Stream<BillingReportLine> billableEdges = projects
- .stream()
- .collect(Collectors.toMap(ProjectDTO::getName, ProjectDTO::getEndpoints))
- .entrySet()
- .stream()
- .flatMap(e -> projectEdges(sbn, e.getKey(), e.getValue()));
- final Stream<BillingReportLine> ssnBillingDataStream = BillingUtils.ssnBillingDataStream(sbn);
- final Stream<BillingReportLine> billableSharedEndpoints = endpointService.getEndpoints()
- .stream()
- .flatMap(endpoint -> BillingUtils.sharedEndpointBillingDataStream(endpoint.getName(), sbn));
-
- billableAdminResources = Stream.of(billableEdges, ssnBillingDataStream, billableSharedEndpoints)
- .flatMap(s -> s);
- }
-
- final Map<String, BillingReportLine> billableResources = Stream.of(billableUserInstances, billingReportLineStream, billableAdminResources)
+ final Map<String, BillingReportLine> billableResources = Stream.of(ssnBillingDataStream, billableEdges, billableSharedEndpoints, billableUserInstances, customImages)
.flatMap(s -> s)
.collect(Collectors.toMap(BillingReportLine::getDlabId, b -> b));
log.debug("Billable resources are: {}", billableResources);
@@ -187,61 +207,62 @@
private Stream<BillingReportLine> projectEdges(String serviceBaseName, String projectName, List<ProjectEndpointDTO> endpoints) {
return endpoints
.stream()
- .flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName(),
- endpoint.getStatus().toString()));
+ .flatMap(endpoint -> BillingUtils.edgeBillingDataStream(projectName, serviceBaseName, endpoint.getName()));
}
- public List<BillingData> getExploratoryRemoteBillingData(UserInfo user, String endpoint, List<UserInstanceDTO> userInstanceDTOS) {
- List<String> dlabIds = null;
- try {
- dlabIds = userInstanceDTOS
- .stream()
- .map(instance -> Stream.concat(BillingUtils.getExploratoryIds(instance.getExploratoryId()).stream(), instance.getResources()
- .stream()
- .map(cr -> BillingUtils.getComputationalIds(cr.getComputationalId()))
- .flatMap(Collection::stream)
- ))
- .flatMap(a -> a)
- .collect(Collectors.toList());
+ private void updateBillingData(EndpointDTO endpointDTO, List<BillingData> billingData) {
+ final String endpointName = endpointDTO.getName();
+ final CloudProvider cloudProvider = endpointDTO.getCloudProvider();
+ final Map<String, BillingReportLine> billableResources = getBillableResources();
+ final Stream<BillingReportLine> billingReportLineStream = billingData
+ .stream()
+ .peek(bd -> bd.setApplication(endpointName))
+ .map(bd -> toBillingReport(bd, getOrDefault(billableResources, bd.getTag())));
- EndpointDTO endpointDTO = endpointService.get(endpoint);
- return provisioningService.get(getBillingUrl(endpointDTO.getUrl(), BILLING_PATH), user.getAccessToken(),
+ if (cloudProvider == CloudProvider.GCP) {
+ final Map<String, List<BillingReportLine>> gcpBillingData = billingReportLineStream
+ .collect(Collectors.groupingBy(bd -> bd.getUsageDate().substring(0, USAGE_DATE_FORMAT.length())));
+ updateGcpBillingData(endpointName, gcpBillingData);
+ } else if (cloudProvider == CloudProvider.AWS) {
+ final Map<String, List<BillingReportLine>> awsBillingData = billingReportLineStream
+ .collect(Collectors.groupingBy(BillingReportLine::getUsageDate));
+ updateAwsBillingData(endpointName, awsBillingData);
+ } else if (cloudProvider == CloudProvider.AZURE) {
+ final List<BillingReportLine> billingReportLines = billingReportLineStream
+ .collect(Collectors.toList());
+ updateAzureBillingData(billingReportLines);
+ }
+ }
+
+ private BillingReportLine getOrDefault(Map<String, BillingReportLine> billableResources, String tag) {
+ return billableResources.getOrDefault(tag, BillingReportLine.builder().dlabId(tag).build());
+ }
+
+ private void updateGcpBillingData(String endpointName, Map<String, List<BillingReportLine>> billingData) {
+ billingData.forEach((usageDate, billingReportLines) -> {
+ billingDAO.deleteByUsageDateRegex(endpointName, usageDate);
+ billingDAO.save(billingReportLines);
+ });
+ }
+
+ private void updateAwsBillingData(String endpointName, Map<String, List<BillingReportLine>> billingData) {
+ billingData.forEach((usageDate, billingReportLines) -> {
+ billingDAO.deleteByUsageDate(endpointName, usageDate);
+ billingDAO.save(billingReportLines);
+ });
+ }
+
+ private void updateAzureBillingData(List<BillingReportLine> billingReportLines) {
+ billingDAO.save(billingReportLines);
+ }
+
+ private List<BillingData> getBillingData(UserInfo userInfo, EndpointDTO e) {
+ try {
+ return provisioningService.get(getBillingUrl(e.getUrl(), BILLING_PATH), userInfo.getAccessToken(),
new GenericType<List<BillingData>>() {
- }, Collections.singletonMap("dlabIds", String.join(",", dlabIds)));
- } catch (Exception e) {
- log.error("Cannot retrieve billing information for {} {}", dlabIds, e.getMessage());
- return Collections.emptyList();
- }
- }
-
- private List<BillingData> getRemoteBillingData(UserInfo userInfo, BillingFilter filter) {
- List<EndpointDTO> endpoints = endpointService.getEndpoints();
- ExecutorService executor = Executors.newFixedThreadPool(endpoints.size());
- List<Callable<List<BillingData>>> callableTasks = new ArrayList<>();
- endpoints.forEach(e -> callableTasks.add(getTask(userInfo, getBillingUrl(e.getUrl(), BILLING_REPORT_PATH), filter)));
-
- List<BillingData> billingData;
- try {
- log.debug("Trying to retrieve billing info for {}", endpoints);
- billingData = executor.invokeAll(callableTasks)
- .stream()
- .map(this::getBillingReportDTOS)
- .flatMap(Collection::stream)
- .collect(Collectors.toList());
- } catch (Exception e) {
- executor.shutdown();
- log.error("Cannot retrieve billing information {}", e.getMessage(), e);
- throw new DlabException("Cannot retrieve billing information", e);
- }
- executor.shutdown();
- return billingData;
- }
-
- private List<BillingData> getBillingReportDTOS(Future<List<BillingData>> s) {
- try {
- return s.get();
- } catch (InterruptedException | ExecutionException e) {
- log.error("Cannot retrieve billing information {}", e.getMessage());
+ });
+ } catch (Exception ex) {
+ log.error("Cannot retrieve billing information for {}. {}", e.getName(), ex.getMessage());
return Collections.emptyList();
}
}
@@ -262,25 +283,25 @@
.toString();
}
- private Callable<List<BillingData>> getTask(UserInfo userInfo, String url, BillingFilter filter) {
- return () -> provisioningService.get(url, userInfo.getAccessToken(),
- new GenericType<List<BillingData>>() {
- },
- Stream.of(new String[][]{
- {"date-start", filter.getDateStart()},
- {"date-end", filter.getDateEnd()},
- {"dlab-id", filter.getDlabId()},
- {"product", String.join(",", filter.getProducts())}
- }).collect(Collectors.toMap(data -> data[0], data -> data[1])));
- }
-
- private Predicate<BillingReportLine> getBillingReportFilter(BillingFilter filter) {
- return br ->
- (CollectionUtils.isEmpty(filter.getUsers()) || filter.getUsers().contains(br.getUser())) &&
- (CollectionUtils.isEmpty(filter.getProjects()) || filter.getProjects().contains(br.getProject())) &&
- (CollectionUtils.isEmpty(filter.getResourceTypes()) || filter.getResourceTypes().contains(String.valueOf(br.getResourceType()))) &&
- (CollectionUtils.isEmpty(filter.getStatuses()) || filter.getStatuses().contains(br.getStatus())) &&
- (CollectionUtils.isEmpty(filter.getShapes()) || filter.getShapes().contains(br.getShape()));
+ private void appendStatuses(BillingReportLine br) {
+ BillingResourceType resourceType = br.getResourceType();
+ if (BillingResourceType.EDGE == resourceType) {
+ projectService.get(br.getProject()).getEndpoints()
+ .stream()
+ .filter(e -> e.getName().equals(br.getResourceName()))
+ .findAny()
+ .ifPresent(e -> br.setStatus(e.getStatus()));
+ } else if (BillingResourceType.EXPLORATORY == resourceType) {
+ exploratoryService.getUserInstance(br.getUser(), br.getProject(), br.getResourceName())
+ .ifPresent(ui -> br.setStatus(UserInstanceStatus.of(ui.getStatus())));
+ } else if (BillingResourceType.COMPUTATIONAL == resourceType) {
+ exploratoryService.getUserInstance(br.getUser(), br.getProject(), br.getExploratoryName(), true)
+ .flatMap(ui -> ui.getResources()
+ .stream()
+ .filter(cr -> cr.getComputationalName().equals(br.getResourceName()))
+ .findAny())
+ .ifPresent(cr -> br.setStatus(UserInstanceStatus.of(cr.getStatus())));
+ }
}
private boolean isFullReport(UserInfo userInfo) {
@@ -294,21 +315,24 @@
}
}
- private BillingReportLine toBillingData(BillingData billingData, BillingReportLine billingReportLine) {
+ private BillingReportLine toBillingReport(BillingData billingData, BillingReportLine billingReportLine) {
return BillingReportLine.builder()
+ .application(billingData.getApplication())
.cost(billingData.getCost())
.currency(billingData.getCurrency())
.product(billingData.getProduct())
.project(billingReportLine.getProject())
- .usageDateTo(billingData.getUsageDateTo())
+ .endpoint(billingReportLine.getEndpoint())
.usageDateFrom(billingData.getUsageDateFrom())
+ .usageDateTo(billingData.getUsageDateTo())
+ .usageDate(billingData.getUsageDate())
.usageType(billingData.getUsageType())
.user(billingReportLine.getUser())
.dlabId(billingData.getTag())
.resourceType(billingReportLine.getResourceType())
.resourceName(billingReportLine.getResourceName())
- .status(billingReportLine.getStatus())
.shape(billingReportLine.getShape())
+ .exploratoryName(billingReportLine.getExploratoryName())
.build();
}
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
index 9c71f76..645f84b 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
@@ -138,7 +138,7 @@
cloudProvider = response.readEntity(CloudProvider.class);
} catch (Exception e) {
log.error("Cannot connect to url '{}'. {}", url, e.getMessage());
- throw new DlabException(String.format("Cannot connect to url '%s'", url), e);
+ throw new DlabException(String.format("Cannot connect to url '%s'. %s", url, e.getMessage()));
}
if (response.getStatus() != 200) {
log.warn("Endpoint url {} is not valid", url);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
index 6a2a615..9f6be91 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ExploratoryServiceImpl.java
@@ -189,7 +189,17 @@
try {
return Optional.of(exploratoryDAO.fetchExploratoryFields(user, project, exploratoryName));
} catch (DlabException e) {
- log.warn("User instance with exploratory name {} for user {} not found.", exploratoryName, user);
+ log.warn("User instance with exploratory {}, project {} for user {} not found.", exploratoryName, project, user);
+ }
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<UserInstanceDTO> getUserInstance(String user, String project, String exploratoryName, boolean includeCompResources) {
+ try {
+ return Optional.of(exploratoryDAO.fetchExploratoryFields(user, project, exploratoryName, includeCompResources));
+ } catch (DlabException e) {
+ log.warn("User instance with exploratory {}, project {} for user {} not found.", exploratoryName, project, user);
}
return Optional.empty();
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ImageExploratoryServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ImageExploratoryServiceImpl.java
index 85ce534..5cb3a64 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ImageExploratoryServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ImageExploratoryServiceImpl.java
@@ -24,9 +24,11 @@
import com.epam.dlab.backendapi.dao.ExploratoryLibDAO;
import com.epam.dlab.backendapi.dao.ImageExploratoryDao;
import com.epam.dlab.backendapi.domain.EndpointDTO;
+import com.epam.dlab.backendapi.domain.ProjectDTO;
import com.epam.dlab.backendapi.resources.dto.ImageInfoRecord;
import com.epam.dlab.backendapi.service.EndpointService;
import com.epam.dlab.backendapi.service.ImageExploratoryService;
+import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.backendapi.util.RequestBuilder;
import com.epam.dlab.constants.ServiceConsts;
import com.epam.dlab.dto.UserInstanceDTO;
@@ -71,10 +73,12 @@
private RequestBuilder requestBuilder;
@Inject
private EndpointService endpointService;
+ @Inject
+ private ProjectService projectService;
@Override
public String createImage(UserInfo user, String project, String exploratoryName, String imageName, String imageDescription) {
-
+ ProjectDTO projectDTO = projectService.get(project);
UserInstanceDTO userInstance = exploratoryDAO.fetchRunningExploratoryFields(user.getName(), project, exploratoryName);
if (imageExploratoryDao.exist(imageName, userInstance.getProject())) {
@@ -105,7 +109,7 @@
EndpointDTO endpointDTO = endpointService.get(userInstance.getEndpoint());
return provisioningService.post(endpointDTO.getUrl() + ExploratoryAPI.EXPLORATORY_IMAGE,
user.getAccessToken(),
- requestBuilder.newExploratoryImageCreate(user, userInstance, imageName, endpointDTO), String.class);
+ requestBuilder.newExploratoryImageCreate(user, userInstance, imageName, endpointDTO, projectDTO), String.class);
}
@Override
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index 6d46d71..b1eac51 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -22,8 +22,8 @@
import com.epam.dlab.auth.UserInfo;
import com.epam.dlab.backendapi.conf.SelfServiceApplicationConfiguration;
import com.epam.dlab.backendapi.dao.BillingDAO;
-import com.epam.dlab.backendapi.dao.EnvDAO;
import com.epam.dlab.backendapi.dao.ExploratoryDAO;
+import com.epam.dlab.backendapi.domain.BillingReport;
import com.epam.dlab.backendapi.domain.EndpointDTO;
import com.epam.dlab.backendapi.domain.ProjectEndpointDTO;
import com.epam.dlab.backendapi.resources.dto.HealthStatusEnum;
@@ -38,7 +38,6 @@
import com.epam.dlab.dto.aws.edge.EdgeInfoAws;
import com.epam.dlab.dto.azure.edge.EdgeInfoAzure;
import com.epam.dlab.dto.base.edge.EdgeInfo;
-import com.epam.dlab.dto.billing.BillingData;
import com.epam.dlab.dto.gcp.edge.EdgeInfoGcp;
import com.epam.dlab.exceptions.DlabException;
import com.google.inject.Inject;
@@ -46,13 +45,11 @@
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
-import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
+import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@@ -62,7 +59,6 @@
private static final String RELEASE_NOTES_FORMAT = "https://github.com/apache/incubator-dlab/blob/%s" +
"/RELEASE_NOTES.md";
private final ExploratoryDAO expDAO;
- private final EnvDAO envDAO;
private final SelfServiceApplicationConfiguration configuration;
private final BillingDAO billingDAO;
private final ProjectService projectService;
@@ -70,11 +66,10 @@
private final BillingService billingService;
@Inject
- public InfrastructureInfoServiceImpl(ExploratoryDAO expDAO, EnvDAO envDAO, SelfServiceApplicationConfiguration configuration,
+ public InfrastructureInfoServiceImpl(ExploratoryDAO expDAO, SelfServiceApplicationConfiguration configuration,
BillingDAO billingDAO, ProjectService projectService, EndpointService endpointService,
BillingService billingService) {
this.expDAO = expDAO;
- this.envDAO = envDAO;
this.configuration = configuration;
this.billingDAO = billingDAO;
this.projectService = projectService;
@@ -88,8 +83,7 @@
try {
Iterable<Document> documents = expDAO.findExploratory(user.getName());
List<EndpointDTO> allEndpoints = endpointService.getEndpoints();
- return StreamSupport.stream(documents.spliterator(),
- false)
+ return StreamSupport.stream(documents.spliterator(), false)
.collect(Collectors.groupingBy(d -> d.getString("project")))
.entrySet()
.stream()
@@ -100,27 +94,24 @@
.anyMatch(endpoint1 -> endpoint1.getName().equals(endpoint.getName())))
.collect(Collectors.toList());
- List<BillingData> collect = e.getValue()
+ List<BillingReport> billingData = e.getValue()
.stream()
- .map(exp -> {
- List<BillingData> exploratoryRemoteBillingData = new ArrayList<>();
- try {
- exploratoryRemoteBillingData = billingService.getExploratoryRemoteBillingData(user, (String) exp.get("endpoint"),
- expDAO.findExploratories(e.getKey(), (String) exp.get("endpoint"), user.getName()));
- } catch (Exception ex) {
- log.error("Cannot retrieve billing information", ex);
- }
- return exploratoryRemoteBillingData;
- })
- .flatMap(Collection::stream)
+ .map(exp ->
+ billingService.getExploratoryBillingData(exp.getString("project"), exp.getString("endpoint"),
+ exp.getString("exploratory_name"),
+ Optional.ofNullable(exp.get("computational_resources")).map(cr -> (List<Document>) cr).get()
+ .stream()
+ .map(cr -> cr.getString("computational_name"))
+ .collect(Collectors.toList()))
+ )
.collect(Collectors.toList());
final Map<String, Map<String, String>> projectEdges =
- endpoints.stream()
- .collect(Collectors.toMap(ProjectEndpointDTO::getName,
- endpointDTO -> getSharedInfo(endpointDTO.getEdgeInfo())));
- return new ProjectInfrastructureInfo(e.getKey(),
- billingDAO.getBillingProjectQuoteUsed(e.getKey()), projectEdges, e.getValue(), collect, endpointResult);
+ endpoints
+ .stream()
+ .collect(Collectors.toMap(ProjectEndpointDTO::getName, this::getSharedInfo));
+ return new ProjectInfrastructureInfo(e.getKey(), billingDAO.getBillingProjectQuoteUsed(e.getKey()),
+ projectEdges, e.getValue(), billingData, endpointResult);
})
.collect(Collectors.toList());
} catch (Exception e) {
@@ -161,18 +152,22 @@
.build();
}
- private Map<String, String> getSharedInfo(EdgeInfo edgeInfo) {
- Map<String, String> shared = new HashMap<>();
- if (Objects.isNull(edgeInfo)) {
- return shared;
+ private Map<String, String> getSharedInfo(ProjectEndpointDTO endpointDTO) {
+ Optional<EdgeInfo> edgeInfo = Optional.ofNullable(endpointDTO.getEdgeInfo());
+ if (!edgeInfo.isPresent()) {
+ return Collections.emptyMap();
}
- shared.put("edge_node_ip", edgeInfo.getPublicIp());
- if (edgeInfo instanceof EdgeInfoAws) {
- EdgeInfoAws edgeInfoAws = (EdgeInfoAws) edgeInfo;
+ EdgeInfo edge = edgeInfo.get();
+ Map<String, String> shared = new HashMap<>();
+
+ shared.put("status", endpointDTO.getStatus().toString());
+ shared.put("edge_node_ip", edge.getPublicIp());
+ if (edge instanceof EdgeInfoAws) {
+ EdgeInfoAws edgeInfoAws = (EdgeInfoAws) edge;
shared.put("user_own_bicket_name", edgeInfoAws.getUserOwnBucketName());
shared.put("shared_bucket_name", edgeInfoAws.getSharedBucketName());
- } else if (edgeInfo instanceof EdgeInfoAzure) {
- EdgeInfoAzure edgeInfoAzure = (EdgeInfoAzure) edgeInfo;
+ } else if (edge instanceof EdgeInfoAzure) {
+ EdgeInfoAzure edgeInfoAzure = (EdgeInfoAzure) edge;
shared.put("user_container_name", edgeInfoAzure.getUserContainerName());
shared.put("shared_container_name", edgeInfoAzure.getSharedContainerName());
shared.put("user_storage_account_name", edgeInfoAzure.getUserStorageAccountName());
@@ -180,8 +175,8 @@
shared.put("datalake_name", edgeInfoAzure.getDataLakeName());
shared.put("datalake_user_directory_name", edgeInfoAzure.getDataLakeDirectoryName());
shared.put("datalake_shared_directory_name", edgeInfoAzure.getDataLakeSharedDirectoryName());
- } else if (edgeInfo instanceof EdgeInfoGcp) {
- EdgeInfoGcp edgeInfoGcp = (EdgeInfoGcp) edgeInfo;
+ } else if (edge instanceof EdgeInfoGcp) {
+ EdgeInfoGcp edgeInfoGcp = (EdgeInfoGcp) edge;
shared.put("user_own_bucket_name", edgeInfoGcp.getUserOwnBucketName());
shared.put("shared_bucket_name", edgeInfoGcp.getSharedBucketName());
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/UserGroupServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/UserGroupServiceImpl.java
index 1758a8b..9eb25c3 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/UserGroupServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/UserGroupServiceImpl.java
@@ -24,6 +24,7 @@
import com.epam.dlab.backendapi.dao.UserRoleDao;
import com.epam.dlab.backendapi.domain.ProjectDTO;
import com.epam.dlab.backendapi.resources.dto.UserGroupDto;
+import com.epam.dlab.backendapi.resources.dto.UserRoleDto;
import com.epam.dlab.backendapi.roles.UserRoles;
import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.backendapi.service.UserGroupService;
@@ -45,6 +46,8 @@
@Slf4j
public class UserGroupServiceImpl implements UserGroupService {
private static final String ROLE_NOT_FOUND_MSG = "Any of role : %s were not found";
+ private static final String ADMIN = "admin";
+ private static final String PROJECT_ADMIN = "projectAdmin";
@Inject
private UserGroupDao userGroupDao;
@@ -97,22 +100,30 @@
@Override
public List<UserGroupDto> getAggregatedRolesByGroup(UserInfo user) {
if (UserRoles.isAdmin(user)) {
- return userRoleDao.aggregateRolesByGroup(true);
+ return userRoleDao.aggregateRolesByGroup();
} else if (UserRoles.isProjectAdmin(user)) {
Set<String> groups = projectService.getProjects(user)
.stream()
.map(ProjectDTO::getGroups)
.flatMap(Collection::stream)
.collect(Collectors.toSet());
- return userRoleDao.aggregateRolesByGroup(false)
+ return userRoleDao.aggregateRolesByGroup()
.stream()
- .filter(userGroup -> groups.contains(userGroup.getGroup()))
+ .filter(userGroup -> groups.contains(userGroup.getGroup()) && !containsAdministrationPermissions(userGroup))
.collect(Collectors.toList());
} else {
throw new DlabException(String.format("User %s doesn't have appropriate permission", user.getName()));
}
}
+ private boolean containsAdministrationPermissions(UserGroupDto userGroup) {
+ List<String> ids = userGroup.getRoles()
+ .stream()
+ .map(UserRoleDto::getId)
+ .collect(Collectors.toList());
+ return ids.contains(ADMIN) || ids.contains(PROJECT_ADMIN);
+ }
+
private void updateGroup(String group, Set<String> roleIds, Set<String> users) {
log.debug("Updating users for group {}: {}", group, users);
userGroupDao.updateUsers(group, users);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
index 2c5b87f..7e46a09 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/BillingUtils.java
@@ -47,7 +47,8 @@
import static com.epam.dlab.dto.billing.BillingResourceType.VOLUME;
public class BillingUtils {
- private static final String[] REPORT_HEADERS = {"DLab ID", "User", "Project", "DLab Resource Type", "Shape", "Product", "Cost"};
+ private static final String[] AVAILABLE_NOTEBOOKS = {"zeppelin", "tensor-rstudio", "rstudio", "tensor", "superset", "jupyterlab", "jupyter", "deeplearning"};
+ private static final String[] REPORT_HEADERS = {"DLab ID", "User", "Project", "DLab Resource Type", "Status", "Shape", "Product", "Cost"};
private static final String REPORT_FIRST_LINE = "Service base name: %s. Available reporting period from: %s to: %s";
private static final String TOTAL_LINE = "Total: %s %s";
private static final String SSN_FORMAT = "%s-ssn";
@@ -63,25 +64,24 @@
private static final String IMAGE_STANDARD_FORMAT1 = "%s-%s-%s-%s-notebook-image";
private static final String IMAGE_STANDARD_FORMAT2 = "%s-%s-%s-notebook-image";
private static final String IMAGE_CUSTOM_FORMAT = "%s-%s-%s-%s-%s";
- private static final String IMAGE_NAME_PREFIX = "docker.dlab-";
- private static final String VOLUME_PRIMARY = "Volume primary";
- private static final String VOLUME_SECONDARY = "Volume secondary";
private static final String SHARED_RESOURCE = "Shared resource";
private static final String IMAGE_NAME = "Image";
private static final String DATAENGINE_NAME_FORMAT = "%d x %s";
- private static final String DATAENGINE_SERVICE_NAME_FORMAT = "Master: %s%sSlave: %d x %s";
+ private static final String DATAENGINE_SERVICE_NAME_FORMAT = "Master: %sSlave: %s";
- public static Stream<BillingReportLine> edgeBillingDataStream(String project, String sbn, String endpoint, String status) {
- final String userEdgeId = String.format(EDGE_FORMAT, sbn, project.toLowerCase(), endpoint);
- final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project.toLowerCase(), endpoint);
- final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project.toLowerCase(), endpoint);
+ public static Stream<BillingReportLine> edgeBillingDataStream(String project, String sbn, String endpoint) {
+ final String userEdgeId = String.format(EDGE_FORMAT, sbn, project, endpoint).toLowerCase();
+ final String edgeVolumeId = String.format(EDGE_VOLUME_FORMAT, sbn, project, endpoint).toLowerCase();
+ final String endpointBucketId = String.format(PROJECT_ENDPOINT_BUCKET_FORMAT, sbn, project, endpoint).toLowerCase();
- return Stream.of(
- BillingReportLine.builder().resourceName("EDGE node").user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(EDGE).status(UserInstanceStatus.of(status)).build(),
+ return Stream.concat(Stream.of(
+ BillingReportLine.builder().resourceName(endpoint).user(SHARED_RESOURCE).project(project).dlabId(userEdgeId).resourceType(EDGE).build(),
BillingReportLine.builder().resourceName("EDGE volume").user(SHARED_RESOURCE).project(project).dlabId(edgeVolumeId).resourceType(VOLUME).build(),
BillingReportLine.builder().resourceName("Project endpoint shared bucket").user(SHARED_RESOURCE).project(project).dlabId(endpointBucketId).resourceType(BUCKET).build()
+ ),
+ standardImageBillingDataStream(sbn, project, endpoint)
);
}
@@ -94,76 +94,92 @@
}
public static Stream<BillingReportLine> sharedEndpointBillingDataStream(String endpoint, String sbn) {
- final String projectEndpointBucketId = String.format(ENDPOINT_SHARED_BUCKET_FORMAT, sbn, endpoint.toLowerCase());
- final String endpointId = String.format(ENDPOINT_FORMAT, sbn, endpoint.toLowerCase());
- return Stream.of(
+ final String projectEndpointBucketId = String.format(ENDPOINT_SHARED_BUCKET_FORMAT, sbn, endpoint).toLowerCase();
+ final String endpointId = String.format(ENDPOINT_FORMAT, sbn, endpoint).toLowerCase();
+ return Stream.concat(Stream.of(
BillingReportLine.builder().resourceName("Endpoint shared bucket").user(SHARED_RESOURCE).project(SHARED_RESOURCE).dlabId(projectEndpointBucketId).resourceType(BUCKET).build(),
BillingReportLine.builder().resourceName("Endpoint").user(SHARED_RESOURCE).project(SHARED_RESOURCE).dlabId(endpointId).resourceType(ENDPOINT).build()
- );
+ ),
+ standardImageBillingDataStream(sbn, endpoint));
}
- public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount, String sbn) {
+ public static Stream<BillingReportLine> exploratoryBillingDataStream(UserInstanceDTO userInstance, Integer maxSparkInstanceCount) {
final Stream<BillingReportLine> computationalStream = userInstance.getResources()
.stream()
.filter(cr -> cr.getComputationalId() != null)
- .flatMap(cr -> Stream.concat(Stream.of(
- withUserProject(userInstance).dlabId(cr.getComputationalId()).resourceName(cr.getComputationalName()).resourceType(COMPUTATIONAL)
- .status(UserInstanceStatus.of(cr.getStatus())).shape(getComputationalShape(cr)).build(),
- withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "m"))
- .resourceType(VOLUME).build(),
- withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_SECONDARY).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "m"))
- .resourceType(VOLUME).build()
- ),
- getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
- ));
- final String exploratoryId = userInstance.getExploratoryId();
- final String imageId1 = String.format(IMAGE_STANDARD_FORMAT1, sbn, userInstance.getProject(), userInstance.getEndpoint(), userInstance.getImageName().replace(IMAGE_NAME_PREFIX, ""));
- final String imageId2 = String.format(IMAGE_STANDARD_FORMAT2, sbn, userInstance.getEndpoint(), userInstance.getImageName().replace(IMAGE_NAME_PREFIX, ""));
+ .flatMap(cr -> {
+ final String computationalId = cr.getComputationalId().toLowerCase();
+ return Stream.concat(Stream.of(
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(computationalId).resourceType(COMPUTATIONAL).shape(getComputationalShape(cr))
+ .exploratoryName(userInstance.getExploratoryName()).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_FORMAT, computationalId)).resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_FORMAT, computationalId)).resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
+ .resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, computationalId, "m"))
+ .resourceType(VOLUME).build()
+ ),
+ getSlaveVolumes(userInstance, cr, maxSparkInstanceCount)
+ );
+ });
+ final String exploratoryName = userInstance.getExploratoryName();
+ final String exploratoryId = userInstance.getExploratoryId().toLowerCase();
final String primaryVolumeId = String.format(VOLUME_PRIMARY_FORMAT, exploratoryId);
final String secondaryVolumeId = String.format(VOLUME_SECONDARY_FORMAT, exploratoryId);
final Stream<BillingReportLine> exploratoryStream = Stream.of(
- withUserProject(userInstance).resourceName(userInstance.getExploratoryName()).dlabId(exploratoryId).resourceType(EXPLORATORY).status(UserInstanceStatus.of(userInstance.getStatus())).shape(userInstance.getShape()).build(),
- BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(imageId1).project(SHARED_RESOURCE).resourceType(IMAGE).build(),
- BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(imageId2).project(SHARED_RESOURCE).resourceType(IMAGE).build(),
- withUserProject(userInstance).resourceName(VOLUME_PRIMARY).dlabId(primaryVolumeId).resourceType(VOLUME).build(),
- withUserProject(userInstance).resourceName(VOLUME_SECONDARY).dlabId(secondaryVolumeId).resourceType(VOLUME).build());
+ withUserProjectEndpoint(userInstance).resourceName(exploratoryName).dlabId(exploratoryId).resourceType(EXPLORATORY).shape(userInstance.getShape()).build(),
+ withUserProjectEndpoint(userInstance).resourceName(exploratoryName).dlabId(primaryVolumeId).resourceType(VOLUME).build(),
+ withUserProjectEndpoint(userInstance).resourceName(exploratoryName).dlabId(secondaryVolumeId).resourceType(VOLUME).build());
+
return Stream.concat(computationalStream, exploratoryStream);
}
public static Stream<BillingReportLine> customImageBillingDataStream(ImageInfoRecord image, String sbn) {
- String imageId = String.format(IMAGE_CUSTOM_FORMAT, sbn, image.getProject(), image.getEndpoint(), image.getApplication(), image.getName());
+ String imageId = String.format(IMAGE_CUSTOM_FORMAT, sbn, image.getProject(), image.getEndpoint(), image.getApplication(), image.getName()).toLowerCase();
return Stream.of(
- BillingReportLine.builder().resourceName(IMAGE_NAME).project(image.getProject()).dlabId(imageId).resourceType(IMAGE).build()
+ BillingReportLine.builder().resourceName(image.getName()).project(image.getProject()).dlabId(imageId).user(image.getUser()).resourceType(IMAGE).build()
);
}
private static Stream<BillingReportLine> getSlaveVolumes(UserInstanceDTO userInstance, UserComputationalResource cr, Integer maxSparkInstanceCount) {
List<BillingReportLine> list = new ArrayList<>();
for (int i = 1; i <= maxSparkInstanceCount; i++) {
- list.add(withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "s" + i))
+ list.add(withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_PRIMARY_COMPUTATIONAL_FORMAT, cr.getComputationalId().toLowerCase(), "s" + i))
.resourceType(VOLUME).build());
- list.add(withUserProject(userInstance).resourceName(cr.getComputationalName() + ":" + VOLUME_PRIMARY).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId(), "s" + i))
+ list.add(withUserProjectEndpoint(userInstance).resourceName(cr.getComputationalName()).dlabId(String.format(VOLUME_SECONDARY_COMPUTATIONAL_FORMAT, cr.getComputationalId().toLowerCase(), "s" + i))
.resourceType(VOLUME).build());
}
return list.stream();
}
- private static BillingReportLine.BillingReportLineBuilder withUserProject(UserInstanceDTO userInstance) {
- return BillingReportLine.builder().user(userInstance.getUser()).project(userInstance.getProject());
+ private static BillingReportLine.BillingReportLineBuilder withUserProjectEndpoint(UserInstanceDTO userInstance) {
+ return BillingReportLine.builder().user(userInstance.getUser()).project(userInstance.getProject()).endpoint(userInstance.getEndpoint());
}
- public static List<String> getComputationalIds(String computationalId) {
- return Arrays.asList(computationalId, String.format(VOLUME_PRIMARY_FORMAT, computationalId));
- }
-
- public static List<String> getExploratoryIds(String exploratoryId) {
- return Arrays.asList(exploratoryId, String.format(VOLUME_PRIMARY_FORMAT, exploratoryId), String.format(VOLUME_SECONDARY_FORMAT, exploratoryId));
- }
-
- private static String getComputationalShape(UserComputationalResource resource) {
+ public static String getComputationalShape(UserComputationalResource resource) {
return DataEngineType.fromDockerImageName(resource.getImageName()) == DataEngineType.SPARK_STANDALONE ?
String.format(DATAENGINE_NAME_FORMAT, resource.getDataengineInstanceCount(), resource.getDataengineShape()) :
- String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), System.lineSeparator(), null, null);
+ String.format(DATAENGINE_SERVICE_NAME_FORMAT, resource.getMasterNodeShape(), resource.getSlaveNodeShape());
+ }
+
+ private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String endpoint) {
+ List<BillingReportLine> list = new ArrayList<>();
+ for (String notebook : AVAILABLE_NOTEBOOKS) {
+ list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT2, sbn, endpoint, notebook).toLowerCase())
+ .user(SHARED_RESOURCE).project(SHARED_RESOURCE).resourceType(IMAGE).build());
+ }
+
+ return list.stream();
+ }
+
+ private static Stream<BillingReportLine> standardImageBillingDataStream(String sbn, String project, String endpoint) {
+ List<BillingReportLine> list = new ArrayList<>();
+ for (String notebook : AVAILABLE_NOTEBOOKS) {
+ list.add(BillingReportLine.builder().resourceName(IMAGE_NAME).dlabId(String.format(IMAGE_STANDARD_FORMAT1, sbn, project, endpoint, notebook).toLowerCase())
+ .project(project).user(SHARED_RESOURCE).resourceType(IMAGE).build());
+ }
+
+ return list.stream();
}
public static String getFirstLine(String sbn, LocalDate from, LocalDate to) {
@@ -173,16 +189,23 @@
CSVFormatter.SEPARATOR, '\"');
}
- public static String getHeader() {
- return CSVFormatter.formatLine(new ArrayList<>(Arrays.asList(BillingUtils.REPORT_HEADERS)), CSVFormatter.SEPARATOR);
+ public static String getHeader(boolean isFull) {
+ List<String> headers = new ArrayList<>(Arrays.asList(BillingUtils.REPORT_HEADERS));
+ if (!isFull) {
+ headers.remove(1);
+ }
+ return CSVFormatter.formatLine(headers, CSVFormatter.SEPARATOR);
}
- public static String printLine(BillingReportLine line) {
+ public static String printLine(BillingReportLine line, boolean isFull) {
List<String> lines = new ArrayList<>();
lines.add(getOrEmpty(line.getDlabId()));
- lines.add(getOrEmpty(line.getUser()));
+ if (isFull) {
+ lines.add(getOrEmpty(line.getUser()));
+ }
lines.add(getOrEmpty(line.getProject()));
- lines.add(getOrEmpty(Optional.ofNullable(line.getResourceType()).map(Enum::name).orElse(null)));
+ lines.add(getOrEmpty(Optional.ofNullable(line.getResourceType()).map(r -> StringUtils.capitalize(r.toString().toLowerCase())).orElse(null)));
+ lines.add(getOrEmpty(Optional.ofNullable(line.getStatus()).map(UserInstanceStatus::toString).orElse(null)));
lines.add(getOrEmpty(line.getShape()));
lines.add(getOrEmpty(line.getProduct()));
lines.add(getOrEmpty(Optional.ofNullable(line.getCost()).map(String::valueOf).orElse(null)));
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/RequestBuilder.java
index 69aeb6c..afe06cd 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/RequestBuilder.java
@@ -509,7 +509,7 @@
@SuppressWarnings("unchecked")
public <T extends ExploratoryImageDTO> T newExploratoryImageCreate(UserInfo userInfo, UserInstanceDTO userInstance,
- String imageName, EndpointDTO endpointDTO) {
+ String imageName, EndpointDTO endpointDTO, ProjectDTO projectDTO) {
checkInappropriateCloudProviderOrElseThrowException(endpointDTO.getCloudProvider());
return (T) newResourceSysBaseDTO(userInfo, endpointDTO.getCloudProvider(), ExploratoryImageDTO.class)
.withProject(userInstance.getProject())
@@ -519,7 +519,8 @@
.withNotebookImage(userInstance.getImageName())
.withImageName(imageName)
.withEndpoint(userInstance.getEndpoint())
- .withTags(userInstance.getTags());
+ .withTags(userInstance.getTags())
+ .withSharedImageEnabled(String.valueOf(projectDTO.isSharedImageEnabled()));
}
@SuppressWarnings("unchecked")
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/management/manage-environment/manage-environment-dilog.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/management/manage-environment/manage-environment-dilog.component.ts
index a3ae8d2..f99944f 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/management/manage-environment/manage-environment-dilog.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/management/manage-environment/manage-environment-dilog.component.ts
@@ -69,7 +69,7 @@
}
public setBudgetLimits(value) {
- if (this.getCurrentTotalValue() >= this.getCurrentUsersTotal()) {
+ if (this.getCurrentTotalValue() >= this.getCurrentUsersTotal() || !this.getCurrentTotalValue()) {
this.dialogRef.close(value);
} else {
this.manageUsersForm.controls['total'].setErrors({ overrun: true });
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.html b/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.html
index 631e7ae..9947516 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.html
@@ -20,7 +20,7 @@
<div class="ani">
<table mat-table [dataSource]="allFilteredEnvironmentData" class="data-grid management mat-elevation-z6">
<ng-container matColumnDef="user">
- <th mat-header-cell *matHeaderCellDef class="user">
+ <th mat-header-cell *matHeaderCellDef class="user label-header">
<span class="label">User</span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -32,7 +32,7 @@
</ng-container>
<ng-container matColumnDef="project">
- <th mat-header-cell *matHeaderCellDef class="project">
+ <th mat-header-cell *matHeaderCellDef class="project label-header">
<span class="label">Project</span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -44,7 +44,7 @@
</ng-container>
<ng-container matColumnDef="type">
- <th mat-header-cell *matHeaderCellDef class="type">
+ <th mat-header-cell *matHeaderCellDef class="type label-header">
<span class="label">Type</span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -56,7 +56,7 @@
</ng-container>
<ng-container matColumnDef="shape">
- <th mat-header-cell *matHeaderCellDef class="shape">
+ <th mat-header-cell *matHeaderCellDef class="shape label-header">
<span class="label">Shape / Resource id</span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -68,7 +68,7 @@
</ng-container>
<ng-container matColumnDef="status">
- <th mat-header-cell *matHeaderCellDef class="status">
+ <th mat-header-cell *matHeaderCellDef class="status label-header">
<span class="label">Status</span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
@@ -77,13 +77,13 @@
<span [hidden]="filtering && filterForm.statuses.length > 0 && !collapsedFilterRow">more_vert</span>
</i>
</button> </th>
- <td mat-cell *matCellDef="let element" class="ani status" >
+ <td mat-cell *matCellDef="let element" class="ani status label-header" >
<span ngClass="{{element.status || ''}}">{{ element.status }}</span>
</td>
</ng-container>
<ng-container matColumnDef="resources">
- <th mat-header-cell *matHeaderCellDef class="resources">
+ <th mat-header-cell *matHeaderCellDef class="resources label-header">
<span class="label">Computational resources</span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -122,7 +122,7 @@
</ng-container>
<ng-container matColumnDef="actions">
- <th mat-header-cell *matHeaderCellDef class="actions">
+ <th mat-header-cell *matHeaderCellDef class="actions label-header">
<span class="label"> Actions </span>
</th>
<td mat-cell *matCellDef="let element" class="settings actions-col">
@@ -166,43 +166,43 @@
<!-- FILTERING -->
<ng-container matColumnDef="user-filter" sticky>
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown (selectionChange)="onUpdate($event)" [type]="'users'" [items]="filterConfiguration.users"
[model]="filterForm.users"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="type-filter" sticky>
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<input placeholder="Filter by environment type" type="text" class="form-control filter-field"
[value]="filterForm.type" (input)="filterForm.type = $event.target.value" />
</th>
</ng-container>
<ng-container matColumnDef="project-filter" sticky>
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown (selectionChange)="onUpdate($event)" [type]="'projects'"
[items]="filterConfiguration.projects" [model]="filterForm.projects"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="shape-filter" sticky>
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown (selectionChange)="onUpdate($event)" [type]="'shapes'"
[items]="filterConfiguration.shapes" [model]="filterForm.shapes"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="status-filter" sticky>
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown (selectionChange)="onUpdate($event)" [type]="'statuses'"
[items]="filterConfiguration.statuses" [model]="filterForm.statuses"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="resource-filter" sticky>
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown (selectionChange)="onUpdate($event)" [type]="'resources'"
[items]="filterConfiguration.resources" [model]="filterForm.resources"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="actions-filter" sticky>
- <th mat-header-cell *matHeaderCellDef class="actions-col">
+ <th mat-header-cell *matHeaderCellDef class="actions-col filter-row-item">
<div class="actions">
<button mat-icon-button class="btn reset" (click)="resetFilterConfigurations()">
<i class="material-icons">close</i>
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.ts
index 216d79c..d0ab9dc 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.ts
@@ -162,11 +162,11 @@
if (action === 'stop') {
this.dialog.open(ConfirmationDialogComponent, {
- data: { notebook: environment, type: type, manageAction: this.isAdmin }, panelClass: 'modal-md'
+ data: { notebook: environment, type: type, manageAction: true }, panelClass: 'modal-md'
}).afterClosed().subscribe(() => this.buildGrid());
} else if (action === 'terminate') {
this.dialog.open(ConfirmationDialogComponent, {
- data: { notebook: environment, type: ConfirmationDialogType.TerminateExploratory, manageAction: this.isAdmin }, panelClass: 'modal-md'
+ data: { notebook: environment, type: ConfirmationDialogType.TerminateExploratory, manageAction: true }, panelClass: 'modal-md'
}).afterClosed().subscribe(() => this.buildGrid());
} else if (action === 'run') {
this.healthStatusService.runEdgeNode().subscribe(() => {
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/project/project.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/project/project.component.ts
index 9833a40..d7442de 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/project/project.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/project/project.component.ts
@@ -122,13 +122,12 @@
private toggleStatusRequest(data, action) {
if ( action === 'terminate') {
- const projectsResources = this.resources
- .filter(resource => resource.project === data.project_name )[0].exploratory
- .filter(expl => expl.status !== 'terminated' && expl.status !== 'terminating');
-
+ const projectsResources = this.resources.filter(resource => resource.project === data.project_name );
+ const activeProjectsResources = projectsResources.length ? projectsResources[0].exploratory
+ .filter(expl => expl.status !== 'terminated' && expl.status !== 'terminating' && expl.status !== 'failed') : [];
let termResources = [];
data.endpoint.forEach(v => {
- termResources = [...termResources, ...projectsResources.filter(resource => resource.endpoint === v)];
+ termResources = [...termResources, ...activeProjectsResources.filter(resource => resource.endpoint === v)];
});
this.dialog.open(NotificationDialogComponent, { data: {
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/roles/roles.component.html b/services/self-service/src/main/resources/webapp/src/app/administration/roles/roles.component.html
index 87e27a2..5c73329 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/roles/roles.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/roles/roles.component.html
@@ -36,7 +36,7 @@
<mat-step [completed]='false'>
<ng-template matStepLabel>Groups</ng-template>
<div class="inner-step mat-reset">
- <input [validator]="groupValidarion()" type="text" placeholder="Enter group name" [(ngModel)]="setupGroup"
+ <input [validator]="groupValidation()" type="text" placeholder="Enter group name" [(ngModel)]="setupGroup"
#setupGroupName="ngModel">
<div class="error" *ngIf="setupGroupName.errors?.patterns && setupGroupName.dirty">Group name can only
contain letters, numbers, hyphens and '_'</div>
@@ -72,7 +72,9 @@
(selectionChange)="onUpdate($event)"
name="roles"
[items]="rolesList"
- [model]="setupRoles">
+ [model]="setupRoles"
+ [isAdmin]="healthStatus?.admin"
+ >
</multi-level-select-dropdown>
</div>
</div>
@@ -104,8 +106,9 @@
(selectionChange)="onUpdate($event)"
[type]="element.group"
[items]="rolesList"
- [model]="element.selected_roles">
-
+ [model]="element.selected_roles"
+ [isAdmin]="healthStatus?.admin"
+ >
</multi-level-select-dropdown>
</div>
</td>
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/roles/roles.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/roles/roles.component.ts
index bf57438..da36c4e 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/roles/roles.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/roles/roles.component.ts
@@ -75,6 +75,9 @@
this.rolesList = roles.map((role) => {
return {role: role.description, type: role.type, cloud: role.cloud};
});
+ if (!this.healthStatus.admin) {
+ this.rolesList = this.rolesList.filter(role => role.role !== 'Allow to execute administration operation');
+ }
this.rolesList = this.rolesList.sort((a, b) => (a.cloud > b.cloud) ? 1 : ((b.cloud > a.cloud) ? -1 : 0));
this.rolesList = this.rolesList.sort((a, b) => (a.type > b.type) ? 1 : ((b.type > a.type) ? -1 : 0));
this.updateGroupData(groups);
@@ -138,10 +141,14 @@
item.selected_roles = [...currGroupSource.selected_roles];
item.roles = [...currGroupSource.roles];
} else {
+ const isSuperAdminGroup = this.startedGroups.filter(v => v.group === item.group)[0].roles.filter(role => role.description === 'Allow to execute administration operation').length;
+ const selectedRoles = isSuperAdminGroup ?
+ [...item.selected_roles.map(v => v.role), 'Allow to execute administration operation'] :
+ item.selected_roles.map(v => v.role);
this.manageRolesGroups({
action, type, value: {
name: item.group,
- roleIds: this.extractIds(this.roles, item.selected_roles.map(v => v.role)),
+ roleIds: this.extractIds(this.roles, selectedRoles),
users: item.users || []
}
});
@@ -208,7 +215,13 @@
return v;
}).sort((a, b) => (a.group > b.group) ? 1 : ((b.group > a.group) ? -1 : 0));
this.groupsData.forEach(item => {
- item.selected_roles = item.roles.map(role => ({role: role.description, type: role.type, cloud: role.cloud}));
+ if (this.healthStatus.admin) {
+ item.selected_roles = item.roles.map(role => ({role: role.description, type: role.type, cloud: role.cloud}));
+ } else {
+ item.selected_roles = item.roles.filter(role => role.description !== 'Allow to execute administration operation')
+ .map(role => ({role: role.description, type: role.type, cloud: role.cloud}));
+ }
+
});
this.getGroupsListCopy();
}
@@ -217,7 +230,7 @@
this.startedGroups = JSON.parse(JSON.stringify(this.groupsData));
}
- public groupValidarion(): ValidatorFn {
+ public groupValidation(): ValidatorFn {
const duplicateList: any = this.groupsData.map(item => item.group.toLowerCase());
return <ValidatorFn>((control: FormControl) => {
if (control.value && duplicateList.includes(CheckUtils.delimitersFiltering(control.value.toLowerCase()))) {
diff --git a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.html b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.html
index 0eb86a4..8413a5d 100644
--- a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.html
@@ -21,7 +21,7 @@
<table mat-table [dataSource]="reportData" class="data-grid reporting mat-elevation-z6">
<ng-container matColumnDef="name">
- <th mat-header-cell *matHeaderCellDef class="env_name">
+ <th mat-header-cell *matHeaderCellDef class="env_name label-header">
<div class="label"><span class="text"> Environment name</span></div>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -30,12 +30,12 @@
</i>
</button>
</th>
- <td mat-cell *matCellDef="let element"> {{element.dlabId}} </td>
+ <td mat-cell *matCellDef="let element"><span class="table-item">{{element.dlabId}}</span></td>
<td mat-footer-cell *matFooterCellDef class="table-footer"></td>
</ng-container>
<ng-container matColumnDef="user">
- <th mat-header-cell *matHeaderCellDef class="th_user">
+ <th mat-header-cell *matHeaderCellDef class="th_user label-header">
<div class="sort">
<div class="sort-arrow up" (click)="sortBy('user', 'down')" [ngClass]="{'active': !!this.active['userdown']}"></div>
<div class="sort-arrow down" (click)="sortBy('user', 'up')" [ngClass]="{'active': !!this.active['userup']}"></div>
@@ -55,7 +55,7 @@
</ng-container>
<ng-container matColumnDef="project">
- <th mat-header-cell *matHeaderCellDef class="th_project">
+ <th mat-header-cell *matHeaderCellDef class="th_project label-header">
<div class="sort">
<div class="sort-arrow up" (click)="sortBy('project', 'down')" [ngClass]="{'active': !!this.active['projectdown']}"></div>
<div class="sort-arrow down" (click)="sortBy('project', 'up')" [ngClass]="{'active': !!this.active['projectup']}"></div>
@@ -73,7 +73,7 @@
</ng-container>
<ng-container matColumnDef="type">
- <th mat-header-cell *matHeaderCellDef class="th_type">
+ <th mat-header-cell *matHeaderCellDef class="th_type label-header">
<div class="label"><span class="text"> Resource Type</span> </div>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -87,7 +87,7 @@
</ng-container>
<ng-container matColumnDef="status">
- <th mat-header-cell *matHeaderCellDef class="th_status">
+ <th mat-header-cell *matHeaderCellDef class="th_status label-header">
<div class="label"><span class="text"> Status</span> </div>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -105,7 +105,7 @@
</ng-container>
<ng-container matColumnDef="shape">
- <th mat-header-cell *matHeaderCellDef class="th_shape">
+ <th mat-header-cell *matHeaderCellDef class="th_shape label-header">
<div class="label"><span class="text"> Instance size</span></div>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -116,13 +116,13 @@
</button>
</th>
<td mat-cell *matCellDef="let element">
- <span>{{element.shape}}</span>
+ <div *ngFor="let shape of shapeSplit(element.shape)">{{shape}}</div>
</td>
<td mat-footer-cell *matFooterCellDef class="table-footer"></td>
</ng-container>
<ng-container matColumnDef="service">
- <th mat-header-cell *matHeaderCellDef class="service">
+ <th mat-header-cell *matHeaderCellDef class="service label-header">
<div class="label"><span class="text"> Product</span> </div>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -140,7 +140,7 @@
</ng-container>
<ng-container matColumnDef="charge" stickyEnd>
- <th mat-header-cell *matHeaderCellDef class="th_charges">
+ <th mat-header-cell *matHeaderCellDef class="th_charges label-header">
<div class="label">
<div class="sort">
<div class="sort-arrow up" (click)="sortBy('cost', 'down')" [ngClass]="{'active': !!this.active['costdown']}"></div>
@@ -161,45 +161,45 @@
<!-- ----------------FILTER -->
<ng-container matColumnDef="name-filter">
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<input #nameFilter type="text" placeholder="Filter by environment name" class="form-control filter-field"
[value]="filtered?.dlab_id" (input)="filteredReportData.dlab_id = $event.target['value']" />
</th>
</ng-container>
<ng-container matColumnDef="user-filter">
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown *ngIf="filterConfiguration" (selectionChange)="onUpdate($event)" [type]="'user'"
[items]="filterConfiguration.users" [model]="filteredReportData.users"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="project-filter">
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown *ngIf="filterConfiguration" (selectionChange)="onUpdate($event)" [type]="'project'"
[items]="filterConfiguration.projects" [model]="filteredReportData.projects"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="type-filter">
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown *ngIf="filterConfiguration" (selectionChange)="onUpdate($event)" [type]="['resource_type']"
[items]="filterConfiguration.resource_type" [model]="filteredReportData.resource_type">
</multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="status-filter">
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown *ngIf="filterConfiguration" (selectionChange)="onUpdate($event)" [type]="'status'"
[items]="filterConfiguration.statuses" [model]="filteredReportData.statuses"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="shape-filter">
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown *ngIf="filterConfiguration" (selectionChange)="onUpdate($event)"
[type]="'shapes'" [items]="filterConfiguration['shapes']"
[model]="filteredReportData['shapes']"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="service-filter">
- <th mat-header-cell *matHeaderCellDef>
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
<multi-select-dropdown *ngIf="filterConfiguration" (selectionChange)="onUpdate($event)"
[type]="['products']"
[items]="filterConfiguration['products']"
@@ -207,8 +207,8 @@
</th>
</ng-container>
<ng-container matColumnDef="actions" stickyEnd>
- <th mat-header-cell *matHeaderCellDef>
- <div class="actions">
+ <th mat-header-cell *matHeaderCellDef class="filter-row-item">
+ <div class="actions th_charges">
<button mat-icon-button class="btn reset" (click)="resetFiltering(); isFiltered = !isFiltered">
<i class="material-icons">close</i>
</button>
diff --git a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.scss b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.scss
index 1e8082b..9c4f819 100644
--- a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.scss
+++ b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.scss
@@ -67,6 +67,9 @@
&.header-row {
th {
font-size: 11px;
+ .label{
+ padding-left: 0;
+ }
}
}
}
@@ -116,9 +119,8 @@
}
.th_charges {
- width: 8%;
- min-width: 140px;
- padding-right: 15px;
+ width: 10%;
+ min-width: 155px;
text-align: right;
.label {
@@ -251,7 +253,7 @@
.env_name,
.service,
.th_type,
- .th_rstatus {
+ .th_status {
width: 10%;
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.ts b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.ts
index 3d99814..ef24283 100644
--- a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.ts
@@ -18,9 +18,7 @@
*/
import {Component, OnInit, Output, EventEmitter, ViewChild, Input} from '@angular/core';
-
-import { DICTIONARY, ReportingConfigModel } from '../../../dictionary/global.dictionary';
-import {logger} from 'codelyzer/util/logger';
+import { ReportingConfigModel } from '../../../dictionary/global.dictionary';
@Component({
selector: 'dlab-reporting-grid',
@@ -30,7 +28,6 @@
})
export class ReportingGridComponent implements OnInit {
- readonly DICTIONARY = DICTIONARY;
filterConfiguration: ReportingConfigModel;
filteredReportData: ReportingConfigModel = new ReportingConfigModel([], [], [], [], [], '', '', '', []);
@@ -46,6 +43,7 @@
@Output() resetRangePicker: EventEmitter<boolean> = new EventEmitter();
displayedColumns: string[] = ['name', 'user', 'project', 'type', 'status', 'shape', 'service', 'charge'];
displayedFilterColumns: string[] = ['name-filter', 'user-filter', 'project-filter', 'type-filter', 'status-filter', 'shape-filter', 'service-filter', 'actions'];
+ filtered: any;
ngOnInit() {}
@@ -68,10 +66,18 @@
sortBy(sortItem, direction) {
let report: Array<object>;
if (direction === 'down') {
- report = this.reportData.sort((a, b) => (a[sortItem] > b[sortItem]) ? 1 : ((b[sortItem] > a[sortItem]) ? -1 : 0));
+ report = this.reportData.sort((a, b) => {
+ if (a[sortItem] === null) a = '';
+ if (b[sortItem] === null) b = '';
+ return (a[sortItem] > b[sortItem]) ? 1 : ((b[sortItem] > a[sortItem]) ? -1 : 0);
+ });
}
if (direction === 'up') {
- report = this.reportData.sort((a, b) => (a[sortItem] < b[sortItem]) ? 1 : ((b[sortItem] < a[sortItem]) ? -1 : 0));
+ report = this.reportData.sort((a, b) => {
+ if (a[sortItem] === null) a = '';
+ if (b[sortItem] === null) b = '';
+ return (a[sortItem] < b[sortItem]) ? 1 : ((b[sortItem] < a[sortItem]) ? -1 : 0)
+ });
}
this.refreshData(this.fullReport, report);
this.removeSorting();
@@ -105,4 +111,8 @@
this.filterReport.emit(this.filteredReportData);
this.resetRangePicker.emit(true);
}
+
+ shapeSplit(shape){
+ return shape.split(/(?=S)/g)
+ }
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts
index 3d04931..70de4ab 100644
--- a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts
@@ -20,8 +20,7 @@
import { Component, OnInit, OnDestroy, ViewChild } from '@angular/core';
import { ToastrService } from 'ngx-toastr';
-
-import {BillingReportService, HealthStatusService} from '../core/services';
+import {ApplicationSecurityService, BillingReportService, HealthStatusService} from '../core/services';
import { ReportingGridComponent } from './reporting-grid/reporting-grid.component';
import { ToolbarComponent } from './toolbar/toolbar.component';
@@ -29,7 +28,6 @@
import { DICTIONARY, ReportingConfigModel } from '../../dictionary/global.dictionary';
import {ProgressBarService} from '../core/services/progress-bar.service';
-
@Component({
selector: 'dlab-reporting',
template: `
@@ -75,10 +73,12 @@
private healthStatusService: HealthStatusService,
public toastr: ToastrService,
private progressBarService: ProgressBarService,
+ private applicationSecurityService: ApplicationSecurityService,
) { }
ngOnInit() {
this.getEnvironmentHealthStatus();
+ this.buildBillingReport();
}
ngOnDestroy() {
@@ -113,17 +113,30 @@
}
rebuildBillingReport(): void {
+ this.checkAutorize();
+ this.buildBillingReport();
+
+ }
+
+ buildBillingReport() {
this.clearStorage();
this.resetRangePicker();
this.reportData.defaultConfigurations();
this.getGeneralBillingData();
}
+ private checkAutorize() {
+ this.applicationSecurityService.isLoggedIn().subscribe( () => {
+ this.getEnvironmentHealthStatus();
+ }
+ );
+ }
+
exportBillingReport(): void {
this.billingReportService.downloadReport(this.reportData)
.subscribe(
data => FileUtils.downloadFile(data),
- error => this.toastr.error('Billing report export failed!', 'Oops!'));
+ () => this.toastr.error('Billing report export failed!', 'Oops!'));
}
getDefaultFilterConfiguration(data): void {
@@ -143,25 +156,22 @@
types.push(item['resource_type']);
if (item.shape && types.indexOf(item.shape)) {
- shapes.push(item.shape);
+ if (item.shape.indexOf('Master') > -1) {
+ for (let shape of item.shape.split(/(?=S)/g)) {
+ shape = shape.replace('Master: ', '');
+ shape = shape.replace(/Slave: /, '');
+ shape = shape.replace(/\s+/g, '');
+ shapes.indexOf(shape) === -1 && shapes.push(shape);
+ }
+ } else if (item.shape.match(/\d x \S+/)) {
+ const parsedShape = item.shape.match(/\d x \S+/)[0].split(' x ')[1];
+ if (shapes.indexOf(parsedShape) === -1) {
+ shapes.push(parsedShape);
+ }
+ } else {
+ shapes.indexOf(item.shape) === -1 && shapes.push(item.shape);
+ }
}
- // if (item.shapes.indexOf('Master') > -1) {
- // for (let shape of item.shapes.split('\n')) {
- // shape = shape.replace('Master: ', '');
- // shape = shape.replace(/Slave:\s+\d+ x /, '');
- // shape = shape.replace(/\s+/g, '');
- //
- // shapes.indexOf(shape) === -1 && shapes.push(shape);
- // }
- // } else if (item.shapes.match(/\d x \S+/)) {
- // const parsedShape = item.shapes.match(/\d x \S+/)[0].split(' x ')[1];
- // if (shapes.indexOf(parsedShape) === -1) {
- // shapes.push(parsedShape);
- // }
- // } else {
- // shapes.indexOf(item.shapes) === -1 && shapes.push(item.shapes);
- // }
- // }
if (item.product && services.indexOf(item.product) === -1)
services.push(item.product);
@@ -199,7 +209,6 @@
.subscribe((result: any) => {
this.billingEnabled = result.billingEnabled;
this.admin = result.admin;
- this.rebuildBillingReport();
});
}
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/cost-details-dialog/cost-details-dialog.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/cost-details-dialog/cost-details-dialog.component.html
index 95ae591..1331f5f 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/cost-details-dialog/cost-details-dialog.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/cost-details-dialog/cost-details-dialog.component.html
@@ -35,24 +35,24 @@
<mat-list>
<mat-list-item class="list-header">
<div class="resource-name ellipsis" [ngClass]="{ 'wide-name-field' : provider === 'azure' }">Name</div>
- <div class="service">{{ DICTIONARY[provider].service }}</div>
- <div class="resource-type" *ngIf="provider === 'aws'">Type</div>
- <div class="cost-currency">Cost</div>
+ <div class="service">Product</div>
+<!-- <div class="resource-type" *ngIf="provider === 'aws'">Type</div>-->
<div class="usage-date-start">Start</div>
<div class="usage-date-end">End</div>
+ <div class="cost-currency">Cost</div>
</mat-list-item>
<div class="scrolling-content" id="scrolling">
- <mat-list-item *ngFor="let item of notebook.billing">
+ <mat-list-item *ngFor="let item of notebook.billing.report_lines">
<div class="resource-name" [ngClass]="{ 'wide-name-field' : provider === 'azure' }"
- matTooltip="{{ item[DICTIONARY[provider].billing.resourceName] }}"
+ matTooltip="{{ item.resource_name }}"
matTooltipPosition="above">
- {{ item[DICTIONARY[provider].billing.resourceName] }}
+ {{ item.resource_name }}
</div>
- <div class="service">{{ item[DICTIONARY[provider].billing.service] }}</div>
- <div class="resource-type" *ngIf="provider === 'aws'">{{ item[DICTIONARY[provider].billing.type] }}</div>
- <div class="cost-currency">{{ item[DICTIONARY[provider].billing.cost] }} {{ item[DICTIONARY[provider].billing.currencyCode] }}</div>
- <div class="usage-date-start">{{ item[DICTIONARY[provider].billing.dateFrom] | date }}</div>
- <div class="usage-date-end">{{ item[DICTIONARY[provider].billing.dateTo] | date }}</div>
+ <div class="service">{{ item.product }}</div>
+<!-- <div class="resource-type" >{{ item.resourse_type }}</div>-->
+ <div class="usage-date-start">{{ item.from | date }}</div>
+ <div class="usage-date-end">{{ item.to | date }}</div>
+ <div class="cost-currency">{{ item.cost }} {{ item.currency }}</div>
</mat-list-item>
</div>
</mat-list>
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/cost-details-dialog/cost-details-dialog.component.scss b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/cost-details-dialog/cost-details-dialog.component.scss
index 64ef43c..18998ea 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/cost-details-dialog/cost-details-dialog.component.scss
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/cost-details-dialog/cost-details-dialog.component.scss
@@ -43,7 +43,7 @@
.resource-name,
.usage-date-start,
.usage-date-end {
- width: 15%;
+ width: 20%;
overflow: hidden;
text-overflow: ellipsis;
padding-right: 10px;
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html
index 965f9d8..69edcc3 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html
@@ -39,7 +39,7 @@
</ng-container> -->
<ng-container matColumnDef="name" sticky>
- <th mat-header-cell *matHeaderCellDef class="name-col">
+ <th mat-header-cell *matHeaderCellDef class="name-col label-header">
<span class="label">Environment name</span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -50,7 +50,7 @@
</th>
</ng-container>
<ng-container matColumnDef="statuses">
- <th mat-header-cell *matHeaderCellDef class="status-col">
+ <th mat-header-cell *matHeaderCellDef class="status-col label-header">
<span class="label"> Status </span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -61,7 +61,7 @@
</th>
</ng-container>
<ng-container matColumnDef="shapes">
- <th mat-header-cell *matHeaderCellDef class="shape-col">
+ <th mat-header-cell *matHeaderCellDef class="shape-col label-header">
<span class="label"> Size </span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -72,12 +72,12 @@
</th>
</ng-container>
<ng-container matColumnDef="tag">
- <th mat-header-cell *matHeaderCellDef class="tag-col">
+ <th mat-header-cell *matHeaderCellDef class="tag-col label-header">
<span class="label"> Tags </span>
</th>
</ng-container>
<ng-container matColumnDef="resources">
- <th mat-header-cell *matHeaderCellDef class="resources-col">
+ <th mat-header-cell *matHeaderCellDef class="resources-col label-header">
<span class="label"> Computational resources </span>
<button mat-icon-button aria-label="More" class="ar" (click)="toggleFilterRow()">
<i class="material-icons">
@@ -88,12 +88,12 @@
</th>
</ng-container>
<ng-container matColumnDef="cost">
- <th mat-header-cell *matHeaderCellDef class="cost-col">
+ <th mat-header-cell *matHeaderCellDef class="cost-col label-header">
<span class="label"> Cost </span>
</th>
</ng-container>
<ng-container matColumnDef="actions" stickyEnd>
- <th mat-header-cell *matHeaderCellDef class="actions-col">
+ <th mat-header-cell *matHeaderCellDef class="actions-col label-header">
<span class="label"> Actions </span>
</th>
</ng-container>
@@ -139,7 +139,7 @@
<td *ngIf="healthStatus?.billingEnabled" class="cost-col">
<span class="total_cost">{{ element.cost || 'N/A' }} {{ element.currency_code || '' }}</span>
<span (click)="element.billing && printCostDetails(element)" class="currency_details"
- [ngClass]="{ 'not-allowed' : !element.billing }">
+ [ngClass]="{ 'not-allowed' : !element.billing.report_lines.length }">
<i class="material-icons">help_outline</i>
</span>
</td>
@@ -168,10 +168,12 @@
</div>
</li>
<li *ngIf="element.status.toLowerCase() === 'stopped' || element.status.toLowerCase() === 'stopping'"
- matTooltip="{{isEdgeNodeStopped(element) ? 'Unable to run notebook if edge node is stopped.' : 'Unable to run notebook until it will be stopped.'}}" matTooltipPosition="above"
- [matTooltipDisabled]="!isResourcesInProgress(element) && element.status.toLowerCase() !== 'stopping' && !isEdgeNodeStopped(element)">
+ matTooltip="{{element.edgeNodeStatus !== 'running' ? 'Unable to run notebook if edge node is not running.' : 'Unable to run notebook until it will be stopped.'}}" matTooltipPosition="above"
+ [matTooltipDisabled]="!isResourcesInProgress(element) && element.status.toLowerCase() !== 'stopping' && element.edgeNodeStatus === 'running'"
+ [ngClass]="{'not-allow': isResourcesInProgress(element) || element.status.toLowerCase() === 'stopping' || element.edgeNodeStatus !== 'running' }"
+ >
<div (click)="exploratoryAction(element, 'run')"
- [ngClass]="{'not-allowed': isResourcesInProgress(element) || element.status.toLowerCase() === 'stopping' || isEdgeNodeStopped(element) }">
+ [ngClass]="{'not-allowed': isResourcesInProgress(element) || element.status.toLowerCase() === 'stopping' || element.edgeNodeStatus !== 'running' }">
<i class="material-icons">play_circle_outline</i>
<span>Run</span>
</div>
@@ -222,37 +224,37 @@
<!-- FILTER START -->
<ng-container matColumnDef="name-filter" sticky>
- <th mat-header-cell *matHeaderCellDef class="name-col">
+ <th mat-header-cell *matHeaderCellDef class="name-col filter-row-item">
<input placeholder="Filter by environment name" type="text" class="form-control filter-field"
- [value]="filterForm.name" (input)="filterForm.name = $event.target.value" />
+ [value]="filterForm.name" (input)="filterForm.name = $event.target['value']" />
</th>
</ng-container>
<ng-container matColumnDef="status-filter">
- <th mat-header-cell *matHeaderCellDef class="status-col">
+ <th mat-header-cell *matHeaderCellDef class="status-col filter-row-item">
<multi-select-dropdown (selectionChange)="onUpdate($event)" [type]="'statuses'"
[items]="filterConfiguration.statuses" [model]="filterForm.statuses"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="shape-filter">
- <th mat-header-cell *matHeaderCellDef class="shape-col">
+ <th mat-header-cell *matHeaderCellDef class="shape-col filter-row-item">
<multi-select-dropdown (selectionChange)="onUpdate($event)"
[type]="'sizes'" [items]="filterConfiguration.shapes"
[model]="filterForm.shapes"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="tag-filter">
- <th mat-header-cell *matHeaderCellDef class="tag-col">
+ <th mat-header-cell *matHeaderCellDef class="tag-col filter-row-item">
</th>
</ng-container>
<ng-container matColumnDef="resource-filter">
- <th mat-header-cell *matHeaderCellDef class="resources-col">
+ <th mat-header-cell *matHeaderCellDef class="resources-col filter-row-item">
<multi-select-dropdown (selectionChange)="onUpdate($event)" [type]="'resources'"
[items]="filterConfiguration.resources" [model]="filterForm.resources"></multi-select-dropdown>
</th>
</ng-container>
<ng-container matColumnDef="cost-filter">
- <th mat-header-cell *matHeaderCellDef class="cost-col">
+ <th mat-header-cell *matHeaderCellDef class="cost-col filter-row-item">
</th>
</ng-container>
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.scss b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.scss
index e09ff3e..d6c0556 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.scss
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.scss
@@ -395,3 +395,7 @@
.content-row{
background-clip: padding-box;
}
+
+.not-allow{
+ cursor: not-allowed !important;
+}
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.ts
index 3052d84..8dfdf4e 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.ts
@@ -151,12 +151,6 @@
return false;
}
- public isEdgeNodeStopped(resource) {
- const currProject = this.projects.filter(proj => proj.name === resource.project);
- const currEdgenodeStatus = currProject[0].endpoints.filter(node => node.name === resource.endpoint)[0].status;
- return currEdgenodeStatus === 'STOPPED' || currEdgenodeStatus === 'STOPPING';
- }
-
public filterActiveInstances(): FilterConfigurationModel {
return (<FilterConfigurationModel | any>Object).assign({}, this.filterConfiguration, {
statuses: SortUtils.activeStatuses(),
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts
index 14a2824..e769dbe 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts
@@ -50,6 +50,7 @@
public project: string,
public endpoint: string,
public tags: any,
+ public edgeNodeStatus: string
) { }
public static loadEnvironments(data: Array<any>) {
@@ -59,36 +60,39 @@
project: value.project,
exploratory: value.exploratory.map(el => {
const provider = el.cloud_provider.toLowerCase();
+ const billing = value.exploratoryBilling.filter(res => res.name === el.exploratory_name)[0];
return new ExploratoryModel(
- provider,
- el.exploratory_name,
- el.template_name,
- el.image,
- el.status,
- el.shape,
- el.computational_resources,
- el.up_time,
- el.exploratory_url,
- value.shared[el.endpoint].edge_node_ip,
- el.private_ip,
- el.exploratory_user,
- el.exploratory_pass,
- value.shared[el.endpoint][DICTIONARY[provider].bucket_name],
- value.shared[el.endpoint][DICTIONARY[provider].shared_bucket_name],
- el.error_message,
- el[DICTIONARY[provider].billing.cost],
- el[DICTIONARY[provider].billing.currencyCode],
- el.billing,
- el.libs,
- value.shared[el.endpoint][DICTIONARY[provider].user_storage_account_name],
- value.shared[el.endpoint][DICTIONARY[provider].shared_storage_account_name],
- value.shared[el.endpoint][DICTIONARY[provider].datalake_name],
- value.shared[el.endpoint][DICTIONARY[provider].datalake_user_directory_name],
- value.shared[el.endpoint][DICTIONARY[provider].datalake_shared_directory_name],
- el.project,
- el.endpoint,
- el.tags,
- )})
+ provider,
+ el.exploratory_name,
+ el.template_name,
+ el.image,
+ el.status,
+ el.shape,
+ el.computational_resources,
+ el.up_time,
+ el.exploratory_url,
+ value.shared[el.endpoint].edge_node_ip,
+ el.private_ip,
+ el.exploratory_user,
+ el.exploratory_pass,
+ value.shared[el.endpoint][DICTIONARY[provider].bucket_name],
+ value.shared[el.endpoint][DICTIONARY[provider].shared_bucket_name],
+ el.error_message,
+ billing ? billing.total_cost : '',
+ billing ? billing.currency : '',
+ billing,
+ el.libs,
+ value.shared[el.endpoint][DICTIONARY[provider].user_storage_account_name],
+ value.shared[el.endpoint][DICTIONARY[provider].shared_storage_account_name],
+ value.shared[el.endpoint][DICTIONARY[provider].datalake_name],
+ value.shared[el.endpoint][DICTIONARY[provider].datalake_user_directory_name],
+ value.shared[el.endpoint][DICTIONARY[provider].datalake_shared_directory_name],
+ el.project,
+ el.endpoint,
+ el.tags,
+ value.shared[el.endpoint].status
+ );
+ })
};
});
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/resources.component.html
index 091ccb7..b705c38 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources.component.html
@@ -63,5 +63,5 @@
</div>
</div>
<mat-divider></mat-divider>
- <resources-grid [projects] = "projects"></resources-grid>
+ <resources-grid></resources-grid>
</div>
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/resources.component.ts
index bab05a7..2587a62 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources.component.ts
@@ -17,14 +17,14 @@
* under the License.
*/
-import { Component, OnInit, ViewChild, OnDestroy } from '@angular/core';
+import { Component, OnInit, ViewChild } from '@angular/core';
import { ToastrService } from 'ngx-toastr';
import { MatDialog } from '@angular/material/dialog';
import { ResourcesGridComponent } from './resources-grid/resources-grid.component';
import { ExploratoryEnvironmentCreateComponent } from './exploratory/create-environment';
import { Exploratory } from './resources-grid/resources-grid.model';
-import { HealthStatusService, ProjectService } from '../core/services';
+import {ApplicationSecurityService, HealthStatusService} from '../core/services';
import { ManageUngitComponent } from './manage-ungit/manage-ungit.component';
import { Project } from './../administration/project/project.component';
@@ -45,12 +45,11 @@
public toastr: ToastrService,
private healthStatusService: HealthStatusService,
private dialog: MatDialog,
- private projectService: ProjectService
+ private applicationSecurityService: ApplicationSecurityService
) { }
ngOnInit() {
this.getEnvironmentHealthStatus();
- this.getProjects();
this.exploratoryEnvironments = this.resourcesGrid.environments;
}
@@ -61,8 +60,7 @@
public refreshGrid(): void {
this.resourcesGrid.buildGrid();
- this.getProjects();
- this.getEnvironmentHealthStatus();
+ this.checkAutorize();
this.exploratoryEnvironments = this.resourcesGrid.environments;
}
@@ -89,8 +87,11 @@
return this.resourcesGrid.activeProject;
}
- private getProjects() {
- this.projectService.getUserProjectsList().subscribe((projects: any) => this.projects = projects);
+ private checkAutorize() {
+ this.applicationSecurityService.isLoggedIn().subscribe( () => {
+ this.getEnvironmentHealthStatus();
+ }
+ );
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.html b/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.html
index 4e41606..420aa09 100644
--- a/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.html
@@ -38,9 +38,11 @@
</li>
<ng-template ngFor let-item [ngForOf]="items" let-i="index">
- <li class="role-label" role="presentation" *ngIf="i === 0 || model && item.type !== items[i - 1].type" (click)="toggleItemsForLable(item.type, $event)">
+ <li class="role-label" role="presentation" *ngIf="i === 0 || model && item.type !== items[i - 1].type" (click)="toggleItemsForLable(item.type, $event)" >
<a href="#" class="list-item" role="menuitem">
- <span class="arrow" [ngClass]="{'rotate-arrow': isOpenCategory[item.type], 'arrow-checked': selectedAllInCattegory(item.type) || selectedSomeInCattegory(item.type)}"></span>
+ <span class="arrow" [ngClass]="{'rotate-arrow': isOpenCategory[item.type], 'arrow-checked': selectedAllInCattegory(item.type) || selectedSomeInCattegory(item.type)}">
+ <i class="material-icons">keyboard_arrow_right</i>
+ </span>
<span class="empty-checkbox" [ngClass]="{'checked': selectedAllInCattegory(item.type) || selectedSomeInCattegory(item.type)}" (click)="toggleselectedCategory($event, model, item.type);$event.stopPropagation()" >
<span class="checked-checkbox" *ngIf="selectedAllInCattegory(item.type)"></span>
<span class="line-checkbox" *ngIf="selectedSomeInCattegory(item.type)"></span>
@@ -49,7 +51,11 @@
</a>
</li>
- <li class="role-item" role="presentation" *ngIf="model && isOpenCategory[item.type] && item.type !== 'COMPUTATIONAL_SHAPE' && item.type !== 'NOTEBOOK_SHAPE'" >
+ <li class="role-item"
+ role="presentation"
+ *ngIf="model && isOpenCategory[item.type] && item.type !== 'COMPUTATIONAL_SHAPE' && item.type !== 'NOTEBOOK_SHAPE'"
+ [hidden]="!isAdmin && item.role === 'Allow to execute administration operation'"
+ >
<a href="#" class="list-item" role="menuitem" (click)="toggleSelectedOptions($event, model, item)">
<span class="empty-checkbox" [ngClass]="{'checked': checkInModel(item.role)}">
<span class="checked-checkbox" *ngIf="checkInModel(item.role)"></span>
@@ -64,12 +70,20 @@
|| model && isOpenCategory[item.type] && item.type === 'COMPUTATIONAL_SHAPE' && item.type !== items[i - 1].type"
>
<a href="#" class="list-item" role="menuitem">
- <span class="arrow" [ngClass]="{'rotate-arrow': isCloudOpen[item.type + item.cloud], 'arrow-checked': selectedAllInCloud(item.type, item.cloud) || selectedSomeInCloud(item.type, item.cloud)}"></span>
- <span class="empty-checkbox" [ngClass]="{'checked': selectedAllInCloud(item.type, item.cloud) || selectedSomeInCloud(item.type, item.cloud)}" (click)="toggleSelectedCloud($event, model, item.type, item.cloud);$event.stopPropagation()" >
+ <span class="arrow" [ngClass]="{'rotate-arrow': isCloudOpen[item.type + item.cloud], 'arrow-checked': selectedAllInCloud(item.type, item.cloud) || selectedSomeInCloud(item.type, item.cloud)}">
+ <i class="material-icons">keyboard_arrow_right</i>
+ </span>
+ <span class="empty-checkbox"
+ [ngClass]="{
+ 'checked': selectedAllInCloud(item.type, item.cloud)
+ || selectedSomeInCloud(item.type, item.cloud)}"
+ (click)="toggleSelectedCloud($event, model, item.type, item.cloud);
+ $event.stopPropagation()"
+ >
<span class="checked-checkbox" *ngIf="selectedAllInCloud(item.type, item.cloud)"></span>
<span class="line-checkbox" *ngIf="selectedSomeInCloud(item.type, item.cloud)"></span>
</span>
- {{item.cloud || 'AWS'}}
+ {{item.cloud}}
</a>
</li>
<li class="role-cloud-item" role="presentation" *ngIf="model && isCloudOpen[item.type + item.cloud] && isOpenCategory[item.type]" >
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.scss b/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.scss
index 5323a24..a066dd5 100644
--- a/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.scss
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.scss
@@ -278,23 +278,22 @@
}
&.arrow{
- width: 16px;
- height: 14px;
- border: 8px solid transparent;
- border-left: 8px solid lightgrey;
- left: 10px;
- top: 12px;
- border-radius: 3px;
-
+ left: 2px;
+ top: 9px;
+ i{
+ color: lightgrey;
+ }
&.rotate-arrow{
transform: rotate(90deg);
transition: .1s ease-in-out;
- top: 15px;
- left: 6px;
+ top: 6px;
+ left: 0;
}
&.arrow-checked{
- border-left: 8px solid #35afd5;
+ i{
+ color: #36afd5
+ }
}
}
}
@@ -319,3 +318,7 @@
}
}
+.d-none{
+ display: none;
+}
+
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.ts b/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.ts
index cabf7d9..5b9c1a9 100644
--- a/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/form-controls/multi-level-select-dropdown/multi-level-select-dropdown.component.ts
@@ -30,6 +30,7 @@
@Input() items: Array<any>;
@Input() model: Array<any>;
@Input() type: string;
+ @Input() isAdmin: boolean;
@Output() selectionChange: EventEmitter<{}> = new EventEmitter();
public isOpenCategory = {
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/confirmation-dialog/confirmation-dialog.model.ts b/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/confirmation-dialog/confirmation-dialog.model.ts
index 1bfcd06..79b0512 100644
--- a/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/confirmation-dialog/confirmation-dialog.model.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/confirmation-dialog/confirmation-dialog.model.ts
@@ -76,9 +76,9 @@
}
private terminateExploratory(): Observable<{}> {
- return this.manageAction
- ? this.manageEnvironmentsService.environmentManagement(this.notebook.user, 'terminate', this.notebook.project, this.notebook.name)
- : this.userResourceService.suspendExploratoryEnvironment(this.notebook, 'terminate');
+ return this.manageAction ? this.manageEnvironmentsService.environmentManagement(
+ this.notebook.user, 'terminate', this.notebook.project, this.notebook.name
+ ) : this.userResourceService.suspendExploratoryEnvironment(this.notebook, 'terminate');
}
private stopEdgeNode(): Observable<{}> {
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/navbar/navbar.component.ts b/services/self-service/src/main/resources/webapp/src/app/shared/navbar/navbar.component.ts
index 952dfd0..1f33caa 100644
--- a/services/self-service/src/main/resources/webapp/src/app/shared/navbar/navbar.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/navbar/navbar.component.ts
@@ -111,7 +111,7 @@
this.subscriptions.add(this.healthStatusService.statusData.pipe(skip(1)).subscribe(result => {
this.healthStatus = result;
result.status && this.checkQuoteUsed(this.healthStatus);
- result.status && !result.projectAssigned && this.checkAssignment(this.healthStatus);
+ result.status && !result.projectAssigned && !result.admin && this.checkAssignment(this.healthStatus);
}));
this.subscriptions.add(timer(0, this.CHECK_ACTIVE_SCHEDULE_TIMEOUT).subscribe(() => this.refreshSchedulerData()));
this.currentUserName = this.getUserName();
diff --git a/services/self-service/src/main/resources/webapp/src/assets/styles/_theme.scss b/services/self-service/src/main/resources/webapp/src/assets/styles/_theme.scss
index 7b48bba..43e9c50 100644
--- a/services/self-service/src/main/resources/webapp/src/assets/styles/_theme.scss
+++ b/services/self-service/src/main/resources/webapp/src/assets/styles/_theme.scss
@@ -662,3 +662,8 @@
}
}
+.filter-row-item, .label-header{
+ box-shadow: inset 0 -1px 0 lightgrey;
+ border-bottom: none !important;
+}
+
diff --git a/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/ImageExploratoryResourceTest.java b/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/ImageExploratoryResourceTest.java
index 38c0e46..d74f94e 100644
--- a/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/ImageExploratoryResourceTest.java
+++ b/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/ImageExploratoryResourceTest.java
@@ -272,7 +272,7 @@
}
private List<ImageInfoRecord> getImageList() {
- ImageInfoRecord imageInfoRecord = new ImageInfoRecord("someName", "someDescription", "someProject", "someEndpoint", "someApp",
+ ImageInfoRecord imageInfoRecord = new ImageInfoRecord("someName", "someDescription", "someProject", "someEndpoint", "someUser", "someApp",
"someFullName", ImageStatus.CREATED);
return Collections.singletonList(imageInfoRecord);
}
diff --git a/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/UserRoleResourceTest.java b/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/UserRoleResourceTest.java
index c335db7..c4e2bd6 100644
--- a/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/UserRoleResourceTest.java
+++ b/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/UserRoleResourceTest.java
@@ -62,7 +62,7 @@
@Test
public void getRoles() {
- when(rolesService.getUserRoles(getUserInfo())).thenReturn(Collections.singletonList(getUserRole()));
+ when(rolesService.getUserRoles()).thenReturn(Collections.singletonList(getUserRole()));
final Response response = resources.getJerseyTest()
.target("/role")
@@ -77,7 +77,7 @@
assertEquals(ROLE_ID, actualRoles.get(0).getId());
assertEquals(MediaType.APPLICATION_JSON, response.getHeaderString(HttpHeaders.CONTENT_TYPE));
- verify(rolesService).getUserRoles(getUserInfo());
+ verify(rolesService).getUserRoles();
verifyNoMoreInteractions(rolesService);
}
diff --git a/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/ImageExploratoryServiceImplTest.java b/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/ImageExploratoryServiceImplTest.java
index 57d0284..e15044b 100644
--- a/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/ImageExploratoryServiceImplTest.java
+++ b/services/self-service/src/test/java/com/epam/dlab/backendapi/service/impl/ImageExploratoryServiceImplTest.java
@@ -24,8 +24,10 @@
import com.epam.dlab.backendapi.dao.ExploratoryLibDAO;
import com.epam.dlab.backendapi.dao.ImageExploratoryDao;
import com.epam.dlab.backendapi.domain.EndpointDTO;
+import com.epam.dlab.backendapi.domain.ProjectDTO;
import com.epam.dlab.backendapi.resources.dto.ImageInfoRecord;
import com.epam.dlab.backendapi.service.EndpointService;
+import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.backendapi.util.RequestBuilder;
import com.epam.dlab.cloud.CloudProvider;
import com.epam.dlab.dto.UserInstanceDTO;
@@ -91,6 +93,8 @@
private RequestBuilder requestBuilder;
@Mock
private EndpointService endpointService;
+ @Mock
+ private ProjectService projectService;
@InjectMocks
private ImageExploratoryServiceImpl imageExploratoryService;
@@ -107,6 +111,7 @@
@Test
public void createImage() {
+ when(projectService.get(anyString())).thenReturn(getProjectDTO());
when(exploratoryDAO.fetchRunningExploratoryFields(anyString(), anyString(), anyString())).thenReturn(userInstance);
when(imageExploratoryDao.exist(anyString(), anyString())).thenReturn(false);
@@ -117,7 +122,7 @@
ExploratoryImageDTO eiDto = new ExploratoryImageDTO();
when(endpointService.get(anyString())).thenReturn(endpointDTO());
when(requestBuilder.newExploratoryImageCreate(any(UserInfo.class), any(UserInstanceDTO.class), anyString(),
- any(EndpointDTO.class))).thenReturn(eiDto);
+ any(EndpointDTO.class), any(ProjectDTO.class))).thenReturn(eiDto);
String expectedUuid = "someUuid";
when(provisioningService.post(anyString(), anyString(), any(ExploratoryImageDTO.class), any()))
@@ -129,15 +134,16 @@
assertNotNull(actualUuid);
assertEquals(expectedUuid, actualUuid);
+ verify(projectService).get(PROJECT);
verify(exploratoryDAO).fetchRunningExploratoryFields(USER, PROJECT, EXPLORATORY_NAME);
verify(exploratoryDAO).updateExploratoryStatus(any(ExploratoryStatusDTO.class));
verify(imageExploratoryDao).exist(imageName, PROJECT);
verify(imageExploratoryDao).save(any(Image.class));
verify(libDAO).getLibraries(USER, PROJECT, EXPLORATORY_NAME);
- verify(requestBuilder).newExploratoryImageCreate(userInfo, userInstance, imageName, endpointDTO());
+ verify(requestBuilder).newExploratoryImageCreate(userInfo, userInstance, imageName, endpointDTO(), getProjectDTO());
verify(endpointService).get(anyString());
verify(provisioningService).post(endpointDTO().getUrl() + "exploratory/image", TOKEN, eiDto, String.class);
- verifyNoMoreInteractions(exploratoryDAO, imageExploratoryDao, libDAO, requestBuilder, endpointService, provisioningService);
+ verifyNoMoreInteractions(projectService, exploratoryDAO, imageExploratoryDao, libDAO, requestBuilder, endpointService, provisioningService);
}
@Test
@@ -170,6 +176,7 @@
@Test
public void createImageWhenMethodNewExploratoryImageCreateThrowsException() {
+ when(projectService.get(anyString())).thenReturn(getProjectDTO());
when(exploratoryDAO.fetchRunningExploratoryFields(anyString(), anyString(), anyString())).thenReturn(userInstance);
when(imageExploratoryDao.exist(anyString(), anyString())).thenReturn(false);
@@ -178,7 +185,7 @@
when(exploratoryDAO.updateExploratoryStatus(any(ExploratoryStatusDTO.class)))
.thenReturn(mock(UpdateResult.class));
doThrow(new DlabException("Cannot create instance of resource class")).when(requestBuilder)
- .newExploratoryImageCreate(any(UserInfo.class), any(UserInstanceDTO.class), anyString(), any(EndpointDTO.class));
+ .newExploratoryImageCreate(any(UserInfo.class), any(UserInstanceDTO.class), anyString(), any(EndpointDTO.class), any(ProjectDTO.class));
when(endpointService.get(anyString())).thenReturn(endpointDTO());
String imageName = "someImageName", imageDescription = "someDescription";
@@ -188,14 +195,15 @@
assertEquals("Cannot create instance of resource class", e.getMessage());
}
+ verify(projectService).get(PROJECT);
verify(exploratoryDAO).fetchRunningExploratoryFields(USER, PROJECT, EXPLORATORY_NAME);
verify(exploratoryDAO).updateExploratoryStatus(any(ExploratoryStatusDTO.class));
verify(imageExploratoryDao).exist(imageName, PROJECT);
verify(imageExploratoryDao).save(any(Image.class));
verify(libDAO).getLibraries(USER, PROJECT, EXPLORATORY_NAME);
- verify(requestBuilder).newExploratoryImageCreate(userInfo, userInstance, imageName, endpointDTO());
+ verify(requestBuilder).newExploratoryImageCreate(userInfo, userInstance, imageName, endpointDTO(), getProjectDTO());
verify(endpointService).get(anyString());
- verifyNoMoreInteractions(exploratoryDAO, imageExploratoryDao, libDAO, requestBuilder, endpointService);
+ verifyNoMoreInteractions(projectService, exploratoryDAO, imageExploratoryDao, libDAO, requestBuilder, endpointService);
}
@Test
@@ -300,7 +308,7 @@
}
private ImageInfoRecord getImageInfoRecord() {
- return new ImageInfoRecord("someName", "someDescription", "someProject", "someEndpoint", "someApp",
+ return new ImageInfoRecord("someName", "someDescription", "someProject", "someEndpoint", "someUser", "someApp",
"someFullName", ImageStatus.CREATED);
}
@@ -337,4 +345,8 @@
private EndpointDTO endpointDTO() {
return new EndpointDTO("test", "url", "", null, EndpointDTO.EndpointStatus.ACTIVE, CloudProvider.AWS);
}
+
+ private ProjectDTO getProjectDTO() {
+ return ProjectDTO.builder().name(PROJECT).build();
+ }
}
