Merge pull request #1219 from apache/DATALAB-2442
[DATALAB-2442] added t-shirt size for gpu types, hid the option for recreate node
diff --git a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
index 23b6908..f2ebd5c 100644
--- a/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
+++ b/infrastructure-provisioning/src/general/files/aws/dataengine-service_description.json
@@ -25,7 +25,6 @@
"templates":
[
{"version":"emr-5.30.0", "applications": [{"Name":"Hadoop", "Version": "2.8.5"}, {"Name":"Spark", "Version": "2.4.5"}, {"Name":"Hive", "Version": "2.3.6"}]},
- {"version":"emr-6.0.0", "applications": [{"Name":"Hadoop", "Version": "3.2.1"}, {"Name":"Spark", "Version": "2.4.4"}, {"Name":"Hive", "Version": "3.1.2"}]},
{"version":"emr-6.2.0", "applications": [{"Name":"Hadoop", "Version": "3.2.1"}, {"Name":"Spark", "Version": "3.0.1"}, {"Name":"Hive", "Version": "3.1.2"}]}
]
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
index 17ac8e0..96d1a3b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/common_prepare_notebook.py
@@ -120,13 +120,14 @@
else notebook_config['expected_primary_image_name'])(str(os.environ.get('notebook_image_name')))
print('Searching pre-configured images')
+ deeplearning_ami = 'false'
+
if os.environ['conf_deeplearning_cloud_ami'] == 'true' and os.environ['application'] == 'deeplearning':
notebook_config['primary_image_name'] = GCPMeta.get_deeplearning_image_by_family(os.environ['notebook_image_name'])
if notebook_config['primary_image_name']:
deeplearning_ami = 'true'
- else:
+ if deeplearning_ami != 'true':
notebook_config['primary_image_name'] = GCPMeta.get_image_by_name(notebook_config['notebook_primary_image_name'])
- deeplearning_ami = 'false'
if notebook_config['primary_image_name'] == '':
notebook_config['primary_image_name'] = os.environ['gcp_{}_image_name'.format(os.environ['conf_os_family'])]
else:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
index 2051b7c..2f08899 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/dataengine_prepare.py
@@ -53,6 +53,11 @@
data_engine['endpoint_tag'] = data_engine['endpoint_name']
data_engine['region'] = os.environ['gcp_region']
data_engine['zone'] = os.environ['gcp_zone']
+ data_engine['gpu_accelerator_type'] = 'None'
+ data_engine['gpu_master_accelerator_type'] = 'None'
+ data_engine['gpu_master_accelerator_count'] = 'None'
+ data_engine['gpu_slave_accelerator_type'] = 'None'
+ data_engine['gpu_slave_accelerator_count'] = 'None'
edge_status = GCPMeta.get_instance_status('{0}-{1}-{2}-edge'.format(data_engine['service_base_name'],
data_engine['project_name'],
@@ -148,7 +153,6 @@
data = {"hostname": data_engine['cluster_name'], "error": ""}
json.dump(data, f)
- data_engine['gpu_accelerator_type'] = 'None'
if os.environ['application'] in ('tensor', 'tensor-rstudio', 'deeplearning'):
if os.environ['gpu_type'] != '':
notebook_config['gpu_accelerator_type'] = os.environ['gpu_type']
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
index cceb651..6830d00 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/deeplearning_configure.py
@@ -267,7 +267,7 @@
try:
# generating output information
ip_address = GCPMeta.get_private_ip_address(notebook_config['instance_name'])
- tensorboard_url = "http://" + ip_address + ":6006/"
+ #tensorboard_url = "http://" + ip_address + ":6006/"
jupyter_ip_url = 'http://' + ip_address + ':8888/{}/'.format(notebook_config['exploratory_name'])
ungit_ip_url = "http://" + ip_address + ":8085/{}-ungit/".format(
notebook_config['exploratory_name'])
@@ -275,8 +275,8 @@
notebook_config['exploratory_name'])
jupyter_ungit_access_url = "https://" + edge_instance_hostname + "/{}-ungit/".format(
notebook_config['exploratory_name'])
- tensorboard_access_url = "https://" + edge_instance_hostname + "/{}-tensor/".format(
- notebook_config['exploratory_name'])
+ #tensorboard_access_url = "https://" + edge_instance_hostname + "/{}-tensor/".format(
+ # notebook_config['exploratory_name'])
print('[SUMMARY]')
logging.info('[SUMMARY]')
print("Instance name: {}".format(notebook_config['instance_name']))
@@ -284,8 +284,8 @@
print("Instance type: {}".format(notebook_config['instance_type']))
print("Key name: {}".format(notebook_config['key_name']))
print("User key name: {}".format(notebook_config['project_name']))
- print("TensorBoard URL: {}".format(tensorboard_url))
- print("TensorBoard log dir: /var/log/tensorboard")
+ #print("TensorBoard URL: {}".format(tensorboard_url))
+ #print("TensorBoard log dir: /var/log/tensorboard")
print("Jupyter URL: {}".format(jupyter_ip_url))
print("Ungit URL: {}".format(ungit_ip_url))
print('SSH access (from Edge node, via IP address): ssh -i {0}.pem {1}@{2}'.format(
@@ -302,8 +302,8 @@
"exploratory_url": [
{"description": "Jupyter",
"url": jupyter_notebook_access_url},
- {"description": "TensorBoard",
- "url": tensorboard_access_url},
+ #{"description": "TensorBoard",
+ # "url": tensorboard_access_url},
{"description": "Ungit",
"url": jupyter_ungit_access_url}#,
#{"description": "Jupyter (via tunnel)",
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/model/exploratory/Exploratory.java b/services/datalab-model/src/main/java/com/epam/datalab/model/exploratory/Exploratory.java
index 1414dd4..31c0e65 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/model/exploratory/Exploratory.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/model/exploratory/Exploratory.java
@@ -33,7 +33,7 @@
private final String version;
private final String templateName;
private final String shape;
- private final String imageName;
+ private String imageName;
private final String endpoint;
private final String project;
private final String exploratoryTag;
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ExploratoryResource.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ExploratoryResource.java
index 23b74a6..39bfb89 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ExploratoryResource.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ExploratoryResource.java
@@ -78,6 +78,7 @@
log.warn("Unauthorized attempt to create a {} by user {}", formDTO.getImage(), userInfo.getName());
throw new DatalabException("You do not have the privileges to create a " + formDTO.getTemplateName());
}
+
String uuid = exploratoryService.create(userInfo, getExploratory(formDTO), formDTO.getProject(), formDTO.getName());
return Response.ok(uuid).build();
@@ -155,8 +156,7 @@
return Exploratory.builder()
.name(formDTO.getName())
.dockerImage(formDTO.getImage())
- .imageName((formDTO.getImageName() == null || formDTO.getImageName().isEmpty())
- ? formDTO.getVersion() : formDTO.getImageName())
+ .imageName(formDTO.getImageName())
.templateName(formDTO.getTemplateName())
.version(formDTO.getVersion())
.clusterConfig(formDTO.getClusterConfig())
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/roles/UserRoles.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/roles/UserRoles.java
index ef161bf..126e465 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/roles/UserRoles.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/roles/UserRoles.java
@@ -261,8 +261,6 @@
if (userRoles == null) {
return true;
}
- LOGGER.info("Check access for user {} with groups {} to {}/{}", userInfo.getName(), userInfo.getRoles(),
- type, name);
Set<String> groups = getGroups(type, name);
if (groups == null || groups.isEmpty()) {
return checkDefault(useDefault);
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ExploratoryServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ExploratoryServiceImpl.java
index 41ce2fd..073e49c 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ExploratoryServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/ExploratoryServiceImpl.java
@@ -118,6 +118,7 @@
try {
final ProjectDTO projectDTO = projectService.get(project);
final EndpointDTO endpointDTO = endpointService.get(exploratory.getEndpoint());
+ changeImageNameForDeepLearningOnAWSandAzure(exploratory, endpointDTO);
final UserInstanceDTO userInstanceDTO = getUserInstanceDTO(userInfo, exploratory, project, endpointDTO.getCloudProvider());
exploratoryDAO.insertExploratory(userInstanceDTO);
isAdded = true;
@@ -143,6 +144,18 @@
}
}
+ private void changeImageNameForDeepLearningOnAWSandAzure(Exploratory exploratory, EndpointDTO endpointDTO) {
+ if (isDeepLearningOnAwsOrAzure(exploratory, endpointDTO)) {
+ if (exploratory.getImageName() == null || exploratory.getImageName().isEmpty())
+ exploratory.setImageName(exploratory.getVersion());
+ }
+ }
+
+ private boolean isDeepLearningOnAwsOrAzure(Exploratory exploratory, EndpointDTO endpointDTO) {
+ return exploratory.getVersion().equals("Deep Learning AMI (Ubuntu 18.04) Version 42.1") ||
+ exploratory.getVersion().equals("microsoft-dsvm:ubuntu-1804:1804-gen2");
+ }
+
@Override
public void updateProjectExploratoryStatuses(UserInfo userInfo, String project, String endpoint, UserInstanceStatus status) {
exploratoryDAO.fetchProjectExploratoriesWhereStatusNotIn(project, endpoint, TERMINATED, FAILED)
