Merge branch 'DATALAB-2091' into DATALAB-2258
diff --git a/infrastructure-provisioning/src/general/files/gcp/jupyter_description.json b/infrastructure-provisioning/src/general/files/gcp/jupyter_description.json
index a94ba57..4c759bb 100644
--- a/infrastructure-provisioning/src/general/files/gcp/jupyter_description.json
+++ b/infrastructure-provisioning/src/general/files/gcp/jupyter_description.json
@@ -29,5 +29,9 @@
"version": "jupyter_notebook-6.0.2",
"vendor": "GCP"
}
+ ],
+ "gpu_types": [
+ {"Size": "S", "Gpu_type": "nvidia-tesla-t4"},
+ {"Size": "M", "Gpu_type": "nvidia-tesla-v100"}
]
}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
index e379901..1e727e7 100644
--- a/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
+++ b/infrastructure-provisioning/src/general/lib/os/debian/edge_lib.py
@@ -128,8 +128,10 @@
with cd('/tmp/src/luarocks-3.3.1/'):
sudo('./configure')
sudo('make install')
+ sudo('luarocks install lua-resty-jwt 0.2.2 --tree /usr/local/openresty/lualib/resty/')
sudo('luarocks install lua-resty-openidc --tree /usr/local/openresty/lualib/resty/')
+ sudo('luarocks install lua-resty-jwt 0.2.2')
sudo('luarocks install lua-resty-openidc')
sudo('useradd -r nginx')
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/UserComputationalResource.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/UserComputationalResource.java
index 8a52ece..b5c8b13 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/UserComputationalResource.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/computational/UserComputationalResource.java
@@ -72,9 +72,20 @@
@JsonProperty("emr_version")
private String awsClusterVersion;
private int totalInstanceCount;
+ @JsonProperty("masterGPUType")
+ private String masterGPUType;
+ @JsonProperty("slaveGPUType")
+ private String slaveGPUType;
+ @JsonProperty("masterGPUCount")
+ private String masterGPUCount;
+ @JsonProperty("slaveGPUCount")
+ private String slaveGPUCount;
+ private boolean enabledGPU;
+
protected List<ClusterConfig> config;
private Map<String, String> tags;
+
public UserComputationalResource(String computationalName, String computationalId, String imageName,
String templateName, String status, Date uptime, SchedulerJobDTO schedulerData,
boolean reuploadKeyRequired, List<ResourceURL> resourceUrl,
diff --git a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/GcpComputationalResource.java b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/GcpComputationalResource.java
index dea7d4d..72485e0 100644
--- a/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/GcpComputationalResource.java
+++ b/services/datalab-model/src/main/java/com/epam/datalab/dto/gcp/computational/GcpComputationalResource.java
@@ -89,5 +89,10 @@
this.slaveGPUCount = slaveGPUCount;
this.slaveGPUType = slaveGPUType;
this.enabledGPU = enabledGPU;
+ super.setMasterGPUCount(masterGPUCount);
+ super.setSlaveGPUCount(slaveGPUCount);
+ super.setMasterGPUType(masterGPUType);
+ super.setSlaveGPUType(slaveGPUType);
+ super.setEnabledGPU(enabledGPU);
}
}
diff --git a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index b7ad081..d758fd4 100644
--- a/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/datalab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -21,6 +21,7 @@
import com.epam.datalab.auth.UserInfo;
import com.epam.datalab.backendapi.conf.SelfServiceApplicationConfiguration;
+import com.epam.datalab.backendapi.dao.ComputationalDAO;
import com.epam.datalab.backendapi.dao.ExploratoryDAO;
import com.epam.datalab.backendapi.domain.BillingReport;
import com.epam.datalab.backendapi.domain.EndpointDTO;
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/bucket-browser.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/bucket-browser.component.ts
index 198152e..7e8ceaf 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/bucket-browser.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/bucket-browser.component.ts
@@ -93,6 +93,7 @@
this.bucketStatus = this.data.bucketStatus;
this.buckets = this.data.buckets;
this.cloud = this.getCloud();
+ // this.cloud = 'azure';
}
ngOnDestroy() {
@@ -128,7 +129,6 @@
this.addedFiles = [];
}
-
public toggleSelectedFile(file, type) {
type === 'file' ? file.isSelected = !file.isSelected : file.isFolderSelected = !file.isFolderSelected;
this.selected = this.folderItems.filter(item => item.isSelected);
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/bucket-data.service.ts b/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/bucket-data.service.ts
index 8573105..bf168ca 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/bucket-data.service.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/bucket-data.service.ts
@@ -146,8 +146,9 @@
parent.children.unshift(name as TodoItemNode);
} else {
if (name) {
+ console.log('parentObject', parent.object);
const child = {item: name, children: [], object: JSON.parse(JSON.stringify(parent.object))};
- child.object.object = child.object.object.slice(0, -1) + child.item + '/';
+ child.object.object = child.object.object.replace(/ا/g, '') + child.item + '/';
parent.children.unshift(child as TodoItemNode);
} else {
parent.children.unshift({item: '', children: [], object: {}} as TodoItemNode);
@@ -166,35 +167,35 @@
public removeItem(parent, child) {
parent.children.splice( parent.children.indexOf(child), 1);
this._bucketData.next(this.data);
- }
+ }
- public processFiles = (files, target, object) => {
- let pointer = target;
- files.forEach((file) => {
- if (!pointer[file]) {
- pointer[file] = {};
- }
- pointer = pointer[file];
- if (!pointer.obj) {
- pointer.obj = object;
- }
-
- });
- }
-
- public processFolderArray = (acc, curr) => {
- const files = curr.object.split('/');
- this.processFiles(files, acc, curr);
-
- return acc;
- }
-
- public convertToFolderTree = (data) => {
- const finalData = data.reduce(this.processFolderArray, {});
- if (Object.keys(finalData).length === 0) {
- return '';
+ public processFiles = (files, target, object) => {
+ let pointer = target;
+ files.forEach((file) => {
+ if (!pointer[file]) {
+ pointer[file] = {};
}
- return finalData;
+ pointer = pointer[file];
+ if (!pointer.obj) {
+ pointer.obj = object;
+ }
+
+ });
+ }
+
+ public processFolderArray = (acc, curr) => {
+ const files = curr.object.split('/');
+ this.processFiles(files, acc, curr);
+
+ return acc;
+ }
+
+ public convertToFolderTree = (data) => {
+ const finalData = data.reduce(this.processFolderArray, {});
+ if (Object.keys(finalData).length === 0) {
+ return '';
}
+ return finalData;
+ }
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/folder-tree/folder-tree.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/folder-tree/folder-tree.component.ts
index 28ed2c0..1721bbc 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/folder-tree/folder-tree.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/bucket-browser/folder-tree/folder-tree.component.ts
@@ -76,9 +76,22 @@
this.dataSource.data = data;
const subject = this.dataSource._flattenedData;
const subjectData = subject.getValue();
+ console.log('selected', this.selectedFolder);
+ console.log('data', subjectData);
if (this.selectedFolder) {
- this.selectedFolder = subjectData.find(v => v.item === this.selectedFolder.item &&
- v.level === this.selectedFolder.level && v.obj === this.selectedFolder.obj);
+ if (this.cloud !== 'azure') {
+ this.selectedFolder = subjectData.find(v => v.item === this.selectedFolder.item &&
+ v.level === this.selectedFolder.level && v.obj === this.selectedFolder.obj);
+ } else {
+ const selectedFolderPath = this.selectedFolder.obj.slice(0, this.selectedFolder.obj.lastIndexOf('/') + 1);
+ this.selectedFolder = subjectData.find(v => {
+ const objectPath = v.obj.slice(0, v.obj.lastIndexOf('/') + 1);
+ console.log('objectPath', selectedFolderPath);
+ console.log('selectedFolderPath', selectedFolderPath);
+ return v.item === this.selectedFolder.item &&
+ v.level === this.selectedFolder.level && objectPath === selectedFolderPath;
+ });
+ }
}
this.expandAllParents(this.selectedFolder || subjectData[0]);
this.showItem(this.selectedFolder || subjectData[0]);
@@ -151,6 +164,7 @@
if (el) {
this.treeControl.expand(el);
this.selectedFolder = el;
+ console.log(this.selectedFolder);
const path = this.getPath(el);
this.path = [];
const data = {
@@ -224,6 +238,9 @@
public removeItem(node: TodoItemFlatNode) {
const parentNode = this.flatNodeMap.get(this.getParentNode(node));
const childNode = this.flatNodeMap.get(node);
+ if (this.cloud === 'azure') {
+ parentNode.object.object = parentNode.object.object.replace(/ا/g, '');
+ }
this.bucketDataService.emptyFolder = null;
this.bucketDataService.removeItem(parentNode!, childNode);
this.resetForm();
@@ -257,6 +274,8 @@
this.toastr.error(error.message || 'Folder creation error!', 'Oops!');
});
} else {
+ flatParent.object.object = flatParent.object.object.replace(/ا/g, '');
+ parent.obj = parent.obj.replace(/ا/g, '');
this.bucketDataService.insertItem(flatParent, itemValue, false);
this.toastr.success('Folder successfully created!', 'Success!');
this.folderCreating = false;
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/computational/cluster-details/cluster-details.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/computational/cluster-details/cluster-details.component.html
index 5ab9801..d4db786 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/computational/cluster-details/cluster-details.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/computational/cluster-details/cluster-details.component.html
@@ -77,7 +77,7 @@
<span *ngIf="DICTIONARY[PROVIDER].cloud_provider === 'gcp' && resource.image === 'docker.datalab-dataengine-service'">1</span>
</div>
</div>
- <div class="row-wrap"
+ <div class="row-wrap"
*ngIf="DICTIONARY[PROVIDER].cloud_provider === 'gcp' && resource.image === 'docker.datalab-dataengine-service'">
<div class="col">
<p>Slave instance number:</p>
@@ -90,13 +90,38 @@
</div>
<div class="col"><span>{{ resource[DICTIONARY[PROVIDER][resource.image].master_node_shape] }}</span></div>
</div>
- <div class="row-wrap" *ngIf="resource.image === 'docker.datalab-dataengine-service'">
+ <div class="row-wrap" *ngIf="resource.image === 'docker.datalab-dataengine-service'">
<div class="col">
<p>Slave instance size:</p>
</div>
<div class="col"><span>{{ resource[DICTIONARY[PROVIDER][resource.image].slave_node_shape] }}</span></div>
</div>
-
+ <ng-template [ngIf]="resource?.enabledGPU">
+ <div class="row-wrap">
+ <div class="col">
+ <p>Master GPU type:</p>
+ </div>
+ <div class="col"><span>{{resource.masterGPUType}}</span></div>
+ </div>
+ <div class="row-wrap">
+ <div class="col">
+ <p>Slave GPU type:</p>
+ </div>
+ <div class="col"><span>{{resource.slaveGPUType}}</span></div>
+ </div>
+ <div class="row-wrap">
+ <div class="col">
+ <p>Master GPU count:</p>
+ </div>
+ <div class="col"><span>{{resource.masterGPUCount}}</span></div>
+ </div>
+ <div class="row-wrap">
+ <div class="col">
+ <p>Slave GPU count:</p>
+ </div>
+ <div class="col"><span>{{resource.slaveGPUCount}}</span></div>
+ </div>
+ </ng-template>
<div *ngIf="resource.status === 'running'">
<div class="row-wrap">
<p class="time_info">Up time {{upTimeInHours}} hour(s) since {{resource.up_time ? (resource.up_time | longDate) : "not specified."
@@ -127,7 +152,6 @@
</p>
</div>
</div>
-
<div class="checkbox-group" *ngIf="resource.image === 'docker.datalab-dataengine'
&& resource.status === 'running'
&& environment.image !== 'docker.datalab-zeppelin'
@@ -167,7 +191,6 @@
</div>
</div>
-
<div class="row-wrap detail-info content-box" *ngIf="resource.error_message">
<p class="failed">{{resource.error_message}}</p>
</div>
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/install-libraries/install-libraries.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/install-libraries/install-libraries.component.html
index 80f81f1..9de8286 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/install-libraries/install-libraries.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/install-libraries/install-libraries.component.html
@@ -86,7 +86,7 @@
</div>
</mat-option>
</ng-template>
- <mat-option *ngIf="model.isEmpty(filteredList) && !validity_format && autoComplete === 'ENABLED'">
+ <mat-option *ngIf="!filteredList?.length && !validity_format && autoComplete === 'ENABLED'">
<span class="configuring">No matches found</span>
</mat-option>
<mat-option *ngIf="validity_format?.length > 0 && autoComplete === 'ENABLED'">
@@ -108,7 +108,7 @@
class="library-input"
[placeholder]="'Enter library version (optional)'"
[(ngModel)]="lib.version"
- [disabled]="!lib.name"
+ [disabled]="!lib.name.trim()"
(keyup)="validateVersion(lib.version)"
(keydown.enter)="addLibrary(lib)"
>
@@ -118,15 +118,15 @@
Library version can only contain Latin letters, numbers and special characters -, _, :, /, ~, ., +.
</span>
<span class="plus-icon"
- [ngClass]="{'not-allow': lib.name?.length < 2
- || (autoComplete === 'ENABLED' && !isLibSelected )
+ [ngClass]="{'not-allow': lib.name?.trim().length < 2
+ || (autoComplete === 'ENABLED' && !isLibSelected && filteredList?.length)
|| this.selectedLib?.isInSelectedList || isVersionInvalid || autoComplete === 'UPDATING'}"
[matTooltip]="this.selectedLib?.isInSelectedList ? 'Library is in selected list' : 'Please select library from autocomplete'"
matTooltipPosition="above" [matTooltipDisabled]="(!this.selectedLib?.isInSelectedList && isLibSelected) || lib.name?.length < 2 || !this.selectedLib?.isInSelectedList && autoComplete === 'NONE'"
>
<mat-icon
(click)="addLibrary(lib)"
- [ngClass]="{'not-allowed': lib.name?.length < 2 || (autoComplete === 'ENABLED' && !isLibSelected)
+ [ngClass]="{'not-allowed': lib.name?.trim().length < 2 || (autoComplete === 'ENABLED' && !isLibSelected && filteredList?.length)
|| this.selectedLib?.isInSelectedList || isVersionInvalid || autoComplete === 'UPDATING'}">add</mat-icon>
</span>
</div>
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/install-libraries/install-libraries.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/install-libraries/install-libraries.component.ts
index 42ed182..45bc6d5 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/install-libraries/install-libraries.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/install-libraries/install-libraries.component.ts
@@ -223,7 +223,7 @@
this.selectedLib = {
name: this.lib.name,
version: this.lib.version,
- isInSelectedList: this.model.selectedLibs.some(el => el.name.toLowerCase() === this.lib.name.toLowerCase())
+ isInSelectedList: this.model.selectedLibs.some(el => el.name.toLowerCase() === this.lib.name.toLowerCase().trim())
};
} else {
this.selectedLib = null;
@@ -231,20 +231,20 @@
}
public addLibrary(item): void {
- if ((this.autoComplete === 'ENABLED' && !this.isLibSelected )
+ if ((this.autoComplete === 'ENABLED' && !this.isLibSelected && this.filteredList?.length)
|| (this.selectedLib && this.selectedLib.isInSelectedList) || this.isVersionInvalid || this.autoComplete === 'UPDATING') {
return;
}
-
+ this.validity_format = '';
this.isLibSelected = false;
if ( (!this.selectedLib && !this.isVersionInvalid) || (!this.selectedLib.isInSelectedList && !this.isVersionInvalid)) {
if ( this.group !== 'java') {
- this.model.selectedLibs.push({ group: this.group, name: item.name, version: item.version || 'N/A' });
+ this.model.selectedLibs.push({ group: this.group, name: item.name.trim(), version: item.version.trim() || 'N/A' });
} else {
this.model.selectedLibs.push({
group: this.group,
name: item.name.substring(0, item.name.lastIndexOf(':')),
- version: item.name.substring(item.name.lastIndexOf(':') + 1) || 'N/A'
+ version: item.name.substring(item.name.lastIndexOf(':') + 1).trim() || 'N/A'
});
}
this.libSearch.setValue('');
@@ -427,10 +427,10 @@
}
private getMatchedLibs() {
- if (!this.lib.name || this.lib.name.length < 2) {
+ if (!this.lib.name || this.lib.name.trim().length < 2) {
return;
}
- this.model.getLibrariesList(this.group, this.lib.name.toLowerCase())
+ this.model.getLibrariesList(this.group, this.lib.name.trim().toLowerCase())
.pipe(
takeUntil(this.unsubscribe$)
)
