Merge branch 'develop' into DATALAB-2347
diff --git a/infrastructure-provisioning/scripts/deploy_datalab.py b/infrastructure-provisioning/scripts/deploy_datalab.py
index e3bdd4c..9eba556 100644
--- a/infrastructure-provisioning/scripts/deploy_datalab.py
+++ b/infrastructure-provisioning/scripts/deploy_datalab.py
@@ -167,6 +167,12 @@
help='password to access repository (used for jars download)')
parser.add_argument('--conf_repository_address', type=str, default='',
help='address to access repository (used for jars download)')
+parser.add_argument('--conf_repository_port', type=str, default='',
+ help='port to access repository (used for jars download)')
+parser.add_argument('--conf_download_jars', type=str, default='false',
+ help='whether to download jars from repository (True) or to build from sources (False)')
+parser.add_argument('--conf_download_docker_images', type=str, default='false',
+ help='whether to download docker images from repository (True) or to build from sources (False)')
parser.add_argument('--conf_letsencrypt_domain_name', type=str, default='', help='Domain names to apply. '
'For multiple domains enter a comma separated list of domains as a parameter'
'ssn.domain_name will be used for ssn_node, DNS A record have to exist during deployment')
@@ -206,11 +212,21 @@
def build_docker_images(args):
- # Building base and ssn docker images
- subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
+ if args.conf_repository_user and args.conf_repository_pass and args.conf_repository_port and args.conf_repository_address and args.conf_download_docker_images == 'true':
+ subprocess.run( 'sudo docker login -u {0} -p {1} {2}:{3}'
+ .format(args.conf_repository_user, args.conf_repository_pass, args.conf_repository_address, args.conf_repository_port), shell=True, check=True)
+ subprocess.run('sudo docker pull {}:{}/docker.datalab-base-{}'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image tag {}:{}/docker.datalab-base-{} docker.datalab-base'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image rm {}:{}/docker.datalab-base-{}'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker pull {}:{}/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image tag {}:{}/docker.datalab-ssn-{} docker.datalab-ssn'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ subprocess.run('sudo docker image rm {}:{}/docker.datalab-ssn-{}'.format(args.conf_repository_address, args.conf_repository_port, args.conf_cloud_provider), shell=True, check=True)
+ else:
+ # Building base and ssn docker images
+ subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --build-arg SRC_PATH="infrastructure-provisioning/src/" --file '
'infrastructure-provisioning/src/general/files/{1}/'
'base_Dockerfile -t docker.datalab-base .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True)
- subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
+ subprocess.run('cd {2}; sudo docker build --build-arg OS={0} --file infrastructure-provisioning/src/general/files/{1}/'
'ssn_Dockerfile -t docker.datalab-ssn .'.format(args.conf_os_family, args.conf_cloud_provider, args.workspace_path), shell=True, check=True)
diff --git a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
index 7218cb9..ec351cf 100644
--- a/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
+++ b/infrastructure-provisioning/src/dataengine/scripts/configure_dataengine.py
@@ -206,7 +206,7 @@
# INSTALL LIVY
if not exists(conn, '/home/{0}/.ensure_dir/livy_ensured'.format(args.os_user)):
conn.sudo('wget -P /tmp/ --user={} --password={} '
- '{}/repository/packages/livy.tar.gz --no-check-certificate'
+ 'https://{}/repository/packages/livy.tar.gz --no-check-certificate'
.format(os.environ['conf_repository_user'],
os.environ['conf_repository_pass'], os.environ['conf_repository_address']))
conn.sudo('tar -xzvf /tmp/livy.tar.gz -C /tmp/')
diff --git a/infrastructure-provisioning/src/general/conf/datalab.ini b/infrastructure-provisioning/src/general/conf/datalab.ini
index f9d2d1b..9b049e6 100644
--- a/infrastructure-provisioning/src/general/conf/datalab.ini
+++ b/infrastructure-provisioning/src/general/conf/datalab.ini
@@ -91,6 +91,12 @@
# repository_pass =
### Repository url
# repository_address =
+### Repository port
+# repository_port =
+### download_jars enabled
+# download_jars =
+### download_docker_images enabled
+# download_docker_images =
### release tag
# release_tag =
### Deeplearning native cloud AMI enabled
diff --git a/infrastructure-provisioning/src/general/lib/os/fab.py b/infrastructure-provisioning/src/general/lib/os/fab.py
index 844d0a7..e005f35 100644
--- a/infrastructure-provisioning/src/general/lib/os/fab.py
+++ b/infrastructure-provisioning/src/general/lib/os/fab.py
@@ -868,7 +868,7 @@
if exists(conn,'/usr/local/lib/livy'):
conn.sudo('rm -r /usr/local/lib/livy')
conn.sudo('wget -P /tmp/ --user={} --password={} '
- '{}/repository/packages/livy.tar.gz --no-check-certificate'
+ 'https://{}/repository/packages/livy.tar.gz --no-check-certificate'
.format(os.environ['conf_repository_user'],
os.environ['conf_repository_pass'], os.environ['conf_repository_address']))
conn.sudo('tar -xzvf /tmp/livy.tar.gz -C /usr/local/lib/')
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index aa20a68..3a1f062 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -116,25 +116,39 @@
host_string = '{}@{}'.format(args.os_user, args.hostname)
if os.environ['conf_cloud_provider'] == 'azure':
conn.local('scp -i {} /root/azure_auth.json {}:{}sources/infrastructure-provisioning/src/base/'
- 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
+ 'azure_auth.json'.format(args.keyfile, host_string, args.datalab_path))
conn.sudo('cp {0}sources/infrastructure-provisioning/src/base/azure_auth.json '
- '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
+ '/home/{1}/keys/azure_auth.json'.format(args.datalab_path, args.os_user))
if region == 'cn-north-1':
add_china_repository(datalab_path)
- for image in image_list:
- name = image['name']
- tag = image['tag']
- conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json '
- '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name))
- if name == 'base':
- conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
- "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} "
- ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
- else:
- conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
- "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
- conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path))
- return True
+ if 'conf_repository_user' in os.environ and 'conf_repository_port' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ and os.environ['conf_download_docker_images'] == 'true':
+ conn.sudo('sudo docker login -u {0} -p {1} {2}:{3}'
+ .format(os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address'], os.environ['conf_repository_port']))
+ for image in image_list:
+ name = image['name']
+ tag = image['tag']
+ conn.sudo('docker pull {0}:{4}/docker.datalab-{2}-{1}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
+ conn.sudo('docker image tag {0}:{4}/docker.datalab-{2}-{1}:{3} docker.datalab-{2}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
+ conn.sudo('docker image rm {0}:{4}/docker.datalab-{2}-{1}:{3}'
+ .format(os.environ['conf_repository_address'], os.environ['conf_cloud_provider'], name, tag, os.environ['conf_repository_port']))
+ return True
+ else:
+ for image in image_list:
+ name = image['name']
+ tag = image['tag']
+ conn.sudo('cp {0}sources/infrastructure-provisioning/src/general/files/{1}/{2}_description.json '
+ '{0}sources/infrastructure-provisioning/src/{2}/description.json'.format(args.datalab_path, args.cloud_provider, name))
+ if name == 'base':
+ conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
+ "--build-arg SRC_PATH=\"\" --file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} "
+ ".'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
+ else:
+ conn.sudo("bash -c 'cd {4}sources/infrastructure-provisioning/src/; docker build --build-arg OS={2} "
+ "--file general/files/{3}/{0}_Dockerfile -t docker.datalab-{0}:{1} .'".format(name, tag, args.os_family, args.cloud_provider, args.datalab_path))
+ conn.sudo('rm -f {}sources/infrastructure-provisioning/src/base/azure_auth.json'.format(args.datalab_path))
+ return True
except:
return False
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
index 7a35f90..d9327aa 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ui.py
@@ -33,6 +33,7 @@
from fabric import *
from patchwork.files import exists
from patchwork import files
+import time
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
@@ -153,22 +154,23 @@
if args.cloud_provider == 'azure' and os.environ['azure_datalake_enable'] == 'true':
conn.sudo('sed -i "s|\'use_ldap\': true|{}|g" ' + args.datalab_path + 'sources/services/self-service/src/main/resources/webapp/src/dictionary/azure.dictionary.ts'.format(
'\'use_ldap\': false'))
-
+ conn.sudo('rm -rf {}sources/services/self-service/src/main/resources/webapp/node_modules'.format(
+ args.datalab_path))
conn.sudo('bash -c "cd {}sources/services/self-service/src/main/resources/webapp/ && echo "N" | npm install"'.format(args.datalab_path))
manage_npm_pkg('bash -c "cd {}sources/services/self-service/src/main/resources/webapp/ && npm run build.prod"'.format(args.datalab_path))
conn.sudo('sudo chown -R {} {}/*'.format(args.os_user, args.datalab_path))
# Building Back-end
- if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ:
+ if 'conf_repository_user' in os.environ and 'conf_repository_pass' in os.environ and 'conf_repository_address' in os.environ and os.environ['conf_download_jars'] == 'true':
conn.sudo(
- 'wget -P {0}sources/services/provisioning-service/target/ --user={1} --password={2} {3}/repository/packages/{4}/provisioning-service-{4}.jar --no-check-certificate'
+ 'wget -P {0}sources/services/provisioning-service/target/ --user={1} --password={2} https://{3}/repository/packages/{4}/provisioning-service-{4}.jar --no-check-certificate'
.format(args.datalab_path, os.environ['conf_repository_user'], os.environ['conf_repository_pass'], os.environ['conf_repository_address'], os.environ['conf_release_tag']))
conn.sudo(
- 'wget -P {0}sources/services/self-service/target/ --user={1} --password={2} {3}/repository/packages/{4}/self-service-{4}.jar --no-check-certificate'
+ 'wget -P {0}sources/services/self-service/target/ --user={1} --password={2} https://{3}/repository/packages/{4}/self-service-{4}.jar --no-check-certificate'
.format(args.datalab_path, os.environ['conf_repository_user'], os.environ['conf_repository_pass'],
os.environ['conf_repository_address'], os.environ['conf_release_tag']))
conn.sudo(
- 'wget -P {0}sources/services/billing-{4}/target/ --user={1} --password={2} {3}/repository/packages/{5}/billing-{4}-{5}.jar --no-check-certificate'
+ 'wget -P {0}sources/services/billing-{4}/target/ --user={1} --password={2} https://{3}/repository/packages/{5}/billing-{4}-{5}.jar --no-check-certificate'
.format(args.datalab_path, os.environ['conf_repository_user'], os.environ['conf_repository_pass'],
os.environ['conf_repository_address'], args.cloud_provider, os.environ['conf_release_tag']))
else:
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java
index 294621e..10eb99e 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/domain/NotebookTemplate.java
@@ -33,7 +33,10 @@
TENSOR_RSTUDIO("RStudio with TensorFlow 2.3.2"),
RSTUDIO("RStudio 1.4.1103"),
TENSOR_GCP("Jupyter with TensorFlow 2.1.0"),
- DEEP_LEARNING_GCP("Deeplearning notebook");
+ DEEP_LEARNING_GCP("Deeplearning notebook"),
+ DEEP_LEARNING_AWS("Deep Learning AMI Version 42.1"),
+ DEEP_LEARNING_AZURE("Data Science Virtual Machine - Ubuntu 18.04");
+
private final String name;
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ProjectResource.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ProjectResource.java
index 15eae6a..040b4f6 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ProjectResource.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/ProjectResource.java
@@ -163,6 +163,7 @@
.build();
}
+
@Operation(summary = "Get project info", tags = "project")
@ApiResponse(responseCode = "200", description = "Return information about project",
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema =
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
index b84247b..b8aa5a6 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/callback/ProjectCallback.java
@@ -87,7 +87,6 @@
private void saveGpuForProject(ProjectResult projectResult, String projectName) {
try {
-
if (projectResult.getEdgeInfo().getGpuList() != null) {
List<String> gpuList = projectResult.getEdgeInfo().getGpuList();
log.info("Adding edgeGpu with gpu_types: {}, for project: {}", gpuList, projectName);
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ProjectActionFormDTO.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ProjectActionFormDTO.java
index 9f6e61e..4fc0b36 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ProjectActionFormDTO.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/resources/dto/ProjectActionFormDTO.java
@@ -33,4 +33,7 @@
@NotNull
@JsonProperty("endpoint")
private final List<String> endpoints;
+ @NotNull
+ @JsonProperty("edge_status")
+ private final String edgeStatus;
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/LibraryServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/LibraryServiceImpl.java
index 9ee95f2..cfc5e6c 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/LibraryServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/LibraryServiceImpl.java
@@ -163,15 +163,13 @@
if (isTemplateGroup(templateName, Stream.of(JUPYTER, ZEPPELIN))) {
groups.addAll(Arrays.asList(GROUP_R_PKG, GROUP_JAVA));
}
- if (isTemplateGroup(templateName, Stream.of(DEEP_LEARNING, TENSOR))) {
+ if (isTemplateGroup(templateName, Stream.of(DEEP_LEARNING, TENSOR, TENSOR_GCP,
+ DEEP_LEARNING_GCP, DEEP_LEARNING_AWS, DEEP_LEARNING_AZURE))) {
groups.add(GROUP_JAVA);
}
if (isTemplateGroup(templateName, Stream.of(RSTUDIO, TENSOR_RSTUDIO))) {
groups.add(GROUP_R_PKG);
}
- if (isTemplateGroup(templateName, Stream.of(DEEP_LEARNING_GCP, TENSOR_GCP))) {
- groups.add(GROUP_JAVA);
- }
return groups
.stream()
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/project/project-list/project-list.component.html b/services/self-service/src/main/resources/webapp/src/app/administration/project/project-list/project-list.component.html
index 27e2ba4..c1f54d1 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/project/project-list/project-list.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/project/project-list/project-list.component.html
@@ -63,7 +63,7 @@
{{ (endpoint.endpointStatus | titlecase) || 'N/A'}}
</span>
</div>
-
+
<span class="status resource-status" [ngClass]="endpoint?.status.toLowerCase() || ''">
{{ endpoint?.status.toLowerCase() }}
</span>
@@ -93,12 +93,12 @@
Stop edge node
</a>
</li>
- <!-- <li class="project-seting-item " *ngIf="element.areTerminatedNode" (click)="openEdgeDialog('recreate', element)">
+ <li class="project-seting-item " *ngIf="element.areTerminatedNode" (click)="openEdgeDialog('recreate', element)">
<i class="material-icons">refresh</i>
<a class="action">
Recreate edge node
</a>
- </li> -->
+ </li>
<li class="project-seting-item " *ngIf="element.areStoppedNode || element.areRunningNode" (click)="openEdgeDialog('terminate', element)">
<i class="material-icons">phonelink_off</i>
<a class="action">
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/project/project.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/project/project.component.ts
index 329c5f5..1409c7b 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/project/project.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/project/project.component.ts
@@ -120,7 +120,8 @@
public toggleStatus($event) {
const data = {
'project_name': $event.project.name,
- endpoint: $event.endpoint.map(endpoint => endpoint.name)
+ endpoint: $event.endpoint.map(endpoint => endpoint.name),
+ 'edge_status': $event.endpoint.map(endpoint => endpoint.status)[0]
};
this.toggleStatusRequest(data, $event.action, $event.oneEdge);
}
diff --git a/services/self-service/src/test/java/com/epam/datalab/backendapi/resources/ProjectResourceTest.java b/services/self-service/src/test/java/com/epam/datalab/backendapi/resources/ProjectResourceTest.java
index 0dee149..96494c8 100644
--- a/services/self-service/src/test/java/com/epam/datalab/backendapi/resources/ProjectResourceTest.java
+++ b/services/self-service/src/test/java/com/epam/datalab/backendapi/resources/ProjectResourceTest.java
@@ -288,7 +288,7 @@
}
private ProjectActionFormDTO getProjectActionDTO() {
- return new ProjectActionFormDTO(PROJECT_NAME, Collections.singletonList(ENDPOINT_NAME));
+ return new ProjectActionFormDTO(PROJECT_NAME, Collections.singletonList(ENDPOINT_NAME),"RUNNING");
}
private UpdateProjectDTO prepareUpdateProjectDTO() {
@@ -296,7 +296,7 @@
}
private ProjectActionFormDTO prepareProjectActionFormDTO() {
- return new ProjectActionFormDTO(PROJECT_NAME, Collections.singletonList(ENDPOINT_NAME));
+ return new ProjectActionFormDTO(PROJECT_NAME, Collections.singletonList(ENDPOINT_NAME),"RUNNING");
}
private List<UpdateProjectBudgetDTO> prepareUpdateProjectBudgetDTOs() {
