Merge branch 'odahu-integration' into develop
diff --git a/infrastructure-provisioning/scripts/deploy_dlab.py b/infrastructure-provisioning/scripts/deploy_dlab.py
index 40b1485..cd27a5b 100644
--- a/infrastructure-provisioning/scripts/deploy_dlab.py
+++ b/infrastructure-provisioning/scripts/deploy_dlab.py
@@ -119,6 +119,19 @@
parser.add_argument('--cost', type=str, default='line_item_blended_cost', help='Column name in report file that contains cost tag')
parser.add_argument('--resource_id', type=str, default='line_item_resource_id', help='Column name in report file that contains '
'dlab resource id tag')
+parser.add_argument('--odahu_tls_crt', type=str, default='', help='TLS Certificate for Odahu cluster')
+parser.add_argument('--odahu_tls_key', type=str, default='', help='TLS Key for Odahu cluster')
+parser.add_argument('--odahu_docker_private_repo', type=str, default='', help='Odahu private repository url')
+parser.add_argument('--odahu_docker_password', type=str, default='', help='Odahu private repository base64 encoded password')
+parser.add_argument('--odahu_keysecret', type=str, default='', help='KeySecret for Odahuflow examples git')
+parser.add_argument('--odahu_oauth_local_jwks', type=str, default='', help='JWKS from keycloak')
+parser.add_argument('--odahu_airflow_secret', type=str, default='', help='Airflow keycloak service account secret')
+parser.add_argument('--odahu_operator_secret', type=str, default='', help='Operator keycloak service account secret')
+parser.add_argument('--odahu_resource_uploader_secret', type=str, default='', help='Resource uploader keycloak service'
+ ' account secret')
+parser.add_argument('--odahu_tester_secret', type=str, default='', help='Tester keycloak service account secret')
+parser.add_argument('--odahu_tester_data_scientist_secret', type=str, default='', help='Tester data scientist keycloak'
+ ' service account secret')
parser.add_argument('--ldap_hostname', type=str, default='localhost', help='Ldap instance hostname')
parser.add_argument('--ldap_dn', type=str, default='dc=example,dc=com',
help='Ldap distinguished name')
@@ -138,6 +151,7 @@
' for billing')
parser.add_argument('--default_endpoint_name', type=str, default='local', help='Name of localhost provisioning service,'
'that created by default')
+parser.add_argument('--ssn_gcr_creds', type=str, default='', help='Base64 encrypted json key for GCR authentication')
parser.add_argument('--conf_stepcerts_enabled', type=str, default='false', help='Enable or disable step certificates')
parser.add_argument('--conf_stepcerts_root_ca', type=str, default='', help='Step root CA')
parser.add_argument('--conf_stepcerts_kid', type=str, default='', help='Step KID')
diff --git a/infrastructure-provisioning/src/general/api/create.py b/infrastructure-provisioning/src/general/api/create.py
index b2437b0..b77cd96 100644
--- a/infrastructure-provisioning/src/general/api/create.py
+++ b/infrastructure-provisioning/src/general/api/create.py
@@ -54,6 +54,14 @@
with open("/response/{}.json".format(os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))
+ elif os.environ['conf_resource'] == 'odahu':
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['odahu_cluster_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['odahu_cluster_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
else:
reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
diff --git a/infrastructure-provisioning/src/general/api/start.py b/infrastructure-provisioning/src/general/api/start.py
index 469d5a6..11ca36f 100644
--- a/infrastructure-provisioning/src/general/api/start.py
+++ b/infrastructure-provisioning/src/general/api/start.py
@@ -48,13 +48,23 @@
reply['response']['result'] = json.loads(f.read())
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
- with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
- os.environ['request_id']), 'w') as response_file:
- response_file.write(json.dumps(reply))
+ if os.environ['conf_resource'] == 'odahu':
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['odahu_cluster_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['odahu_cluster_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
+ else:
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['project_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
try:
local('chmod 666 /response/*')
diff --git a/infrastructure-provisioning/src/general/api/stop.py b/infrastructure-provisioning/src/general/api/stop.py
index 8dc2a11..39d1f8e 100644
--- a/infrastructure-provisioning/src/general/api/stop.py
+++ b/infrastructure-provisioning/src/general/api/stop.py
@@ -49,13 +49,22 @@
except:
reply['response']['result'] = {"error": "Failed to open result.json"}
- reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
- os.environ['project_name'],
- os.environ['request_id'])
+ if os.environ['conf_resource'] == 'odahu':
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['odahu_cluster_name'],
+ os.environ['request_id'])
- with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
- os.environ['request_id']), 'w') as response_file:
- response_file.write(json.dumps(reply))
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['odahu_cluster_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
+ else:
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['project_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
try:
local('chmod 666 /response/*')
diff --git a/infrastructure-provisioning/src/general/api/terminate.py b/infrastructure-provisioning/src/general/api/terminate.py
index 933af27..f637b51 100644
--- a/infrastructure-provisioning/src/general/api/terminate.py
+++ b/infrastructure-provisioning/src/general/api/terminate.py
@@ -54,6 +54,14 @@
with open("/response/{}.json".format(os.environ['request_id']), 'w') as response_file:
response_file.write(json.dumps(reply))
+ elif os.environ['conf_resource'] == 'odahu':
+ reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
+ os.environ['odahu_cluster_name'],
+ os.environ['request_id'])
+
+ with open("/response/{}_{}_{}.json".format(os.environ['conf_resource'], os.environ['odahu_cluster_name'],
+ os.environ['request_id']), 'w') as response_file:
+ response_file.write(json.dumps(reply))
else:
reply['response']['log'] = "/var/log/dlab/{0}/{0}_{1}_{2}.log".format(os.environ['conf_resource'],
os.environ['project_name'],
diff --git a/infrastructure-provisioning/src/general/conf/dlab.ini b/infrastructure-provisioning/src/general/conf/dlab.ini
index 8ab5f9e..e3db900 100644
--- a/infrastructure-provisioning/src/general/conf/dlab.ini
+++ b/infrastructure-provisioning/src/general/conf/dlab.ini
@@ -53,7 +53,7 @@
### CIDR of second VPC
vpc2_cidr = '172.32.0.0/16'
### Enable or disable duo VPC mode(true|false)
-duo_vpc_enable = false
+duo_vpc_enable = falseF
### Range of subnets which will be using for user's environments
# user_subnets_range =
### Comma-separated CIDR of IPs which will have access to SSN and Edge nodes
@@ -210,6 +210,12 @@
# subdomain =
### Role ARN for creating Route53 record
# assume_role_arn =
+### URL to Nexus repository
+nexus_url = ec2-35-163-186-140.us-west-2.compute.amazonaws.com:8083
+### Username for GCR authentication
+# gcr_username =
+### Base64 encrypted json key for GCR authentication
+# gcr_creds =
#--- [edge] section contains all parameters that are using for edge node provisioning ---#
[edge]
@@ -327,6 +333,87 @@
### Depending on RAM size on instance, this parameter determines size of RAM when explicit allocation RAM is used
expl_instance_memory = 8000
+#--- [odahu] odahuflow parameters ---#
+[odahu]
+###
+allowed_cidr = 0.0.0.0/0
+###
+dns_zone_name = develop-dlabanalytics-com
+###
+docker_repo = odahu
+###
+cidr = 172.31.0.0/24
+###
+grafana_admin = grafana_admin
+###
+initial_node_count = 6
+###
+istio_helm_repo = https://storage.googleapis.com/istio-release/releases/1.2.2/charts
+###
+k8s_version = 1.14
+###
+helm_repo = https://raw.githubusercontent.com/odahu/odahu-helm/master
+###
+jupyterhub_enabled = true
+###
+automation_version = 1.1.0
+###
+infra_version = 1.1.0
+###
+ui_version = 1.1.0
+###
+examples_version = 1.1.0
+###
+odahuflow_version = 1.1.0
+###
+mlflow_toolchain_version = 1.1.0
+###
+jupyterlab_version = 1.1.0
+###
+packager_version = 1.1.0
+###
+private_subnet_cidrs = 172.10.2.0/24,172.10.4.0/24
+###
+public_subnet_cidrs = 172.10.1.0/24,172.10.3.0/24
+###
+oauth_mesh_enabled = true
+###
+node_version = 1.13.12-gke.2
+###
+pods_cidr = 10.42.0.0/17
+###
+root_domain = develop.dlabanalytics.com
+###
+service_cidr = 10.42.128.0/20
+###
+dns_project_id = or2-msq-epmc-dlab-t1iylu
+###
+infra_vpc_peering = 0
+### Link to Odahuflow deploy image
+deploy_image = gcr.io/or2-msq-epmd-legn-t1iylu/odahu/odahu-flow-automation:1.1.0
+###
+opa_policy = cGFja2FnZSBvZGFodQoKIyByb2xlcy5yZWdvCgphZG1pbiA6PSAiYWRtaW4iCmRhdGFfc2NpZW50aXN0IDo9ICJkYXRhX3NjaWVudGlzdCIKCiMgcmJhYy5yZWdvCgpyYmFjIDo9IHsKCWRhdGFfc2NpZW50aXN0OiBbCiAgICAJWyIqIiwgImFwaS92MS9tb2RlbC9kZXBsb3ltZW50LioiXSwKICAgIAlbIioiLCAiYXBpL3YxL21vZGVsL3BhY2thZ2luZy4qIl0sCiAgICAJWyIqIiwgImFwaS92MS9tb2RlbC90cmFpbmluZy4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3YxL2Nvbm5lY3Rpb24uKiJdLAogICAgCVsiUE9TVCIsICJhcGkvdjEvY29ubmVjdGlvbi4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3YxL3BhY2thZ2luZy9pbnRlZ3JhdGlvbi4qIl0sCiAgICAJWyJHRVQiLCAiYXBpL3YxL3Rvb2xjaGFpbi9pbnRlZ3JhdGlvbi4qIl0KICAgIF0sCiAgYWRtaW4gOiBbCiAgICAgIFsiLioiLCAiLioiXQogIF0KfQoKIyBpbnB1dF9tYXBwZXIKCnJvbGVzX21hcCA9IHsKCSJvZGFodV9hZG1pbiI6IGFkbWluLAogICAgIm9kYWh1X2RhdGFfc2NpZW50aXN0IjogZGF0YV9zY2llbnRpc3QKfQoKand0ID0gaW5wdXQuYXR0cmlidXRlcy5tZXRhZGF0YV9jb250ZXh0LmZpbHRlcl9tZXRhZGF0YVsiZW52b3kuZmlsdGVycy5odHRwLmp3dF9hdXRobiJdLmZpZWxkcy5qd3RfcGF5bG9hZAoKa2V5Y2xvYWtfdXNlcl9yb2xlc1tyb2xlXXsKCXJvbGUgPSBqd3QuS2luZC5TdHJ1Y3RWYWx1ZS5maWVsZHMucmVhbG1fYWNjZXNzLktpbmQuU3RydWN0VmFsdWUuZmllbGRzLnJvbGVzLktpbmQuTGlzdFZhbHVlLnZhbHVlc1tfXS5LaW5kLlN0cmluZ1ZhbHVlCn0KCnVzZXJfcm9sZXNbcm9sZV17Cglyb2xlID0gcm9sZXNfbWFwW2tleWNsb2FrX3VzZXJfcm9sZXNbX11dCn0KCgpwYXJzZWRfaW5wdXQgPSB7CgkiYWN0aW9uIjogaW5wdXQuYXR0cmlidXRlcy5yZXF1ZXN0Lmh0dHAubWV0aG9kLAogICJyZXNvdXJjZSI6IGlucHV0LmF0dHJpYnV0ZXMucmVxdWVzdC5odHRwLnBhdGgsCiAgInVzZXIiOiB7CiAgICAicm9sZXMiOiB1c2VyX3JvbGVzCiAgfQp9CgoKIyBjb3JlCgpkZWZhdWx0IGFsbG93ID0gZmFsc2UKYWxsb3cgewoJYW55X3VzZXJfcm9sZSA6PSBwYXJzZWRfaW5wdXQudXNlci5yb2xlc1tfXQogICAgYW55X3Blcm1pc3Npb25fb2ZfdXNlcl9yb2xlIDo9IHJiYWNbYW55X3VzZXJfcm9sZV1bX10KICAgIGFjdGlvbiA6PSBhbnlfcGVybWlzc2lvbl9vZl91c2VyX3JvbGVbMF0KICAgIHJlc291cmNlIDo9IGFueV9wZXJtaXNzaW9uX29mX3VzZXJfcm9sZVsxXQoKICAgIHJlX21hdGNoKGFjdGlvbiwgcGFyc2VkX2lucHV0LmFjdGlvbikKICAgIHJlX21hdGNoKHJlc291cmNlLCBwYXJzZWRfaW5wdXQucmVzb3VyY2UpCn0=
+### TLS Certificate for Odahu cluster
+# tls_crt =
+### TLS Key for Odahu cluster
+# tls_key =
+### Odahu private repository base64 encoded password
+# docker_password =
+### KeySecret for Odahuflow examples git
+# keysecret =
+### JWKS from keycloak
+# oauth_local_jwks =
+### Airflow keycloak service account secret
+# airflow_secret =
+### Operator keycloak service account secret
+# operator_secret =
+### Resource uploader keycloak service account secret
+# resource_uploader_secret =
+### Tester keycloak service account secret
+# tester_secret =
+### Tester data scientist keycloak service account secret
+# tester_data_scientist_secret =
+
#--- [ldap] ldap parameters ---#
[ldap]
### Ldap hostname
diff --git a/infrastructure-provisioning/src/general/files/aws/odahu_Dockerfile b/infrastructure-provisioning/src/general/files/aws/odahu_Dockerfile
new file mode 100644
index 0000000..d41a94a
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/aws/odahu_Dockerfile
@@ -0,0 +1,85 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+FROM ODAHU_IMAGE
+ARG OS
+ARG SRC_PATH
+
+ENV PROFILE=/profile.json
+
+# Install any .deb dependecies
+RUN apt-get update && \
+ apt-get -y upgrade && \
+ apt-get -y install python-pip python-dev build-essential python3 python3-dev && \
+ apt-get -y install groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
+ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install any python dependencies
+RUN pip install -UI pip==9.0.3 && \
+ pip install boto3 backoff fabric==1.14.0 fabvenv awscli argparse requests ujson jupyter pycrypto
+
+# Configuring ssh for user
+RUN mkdir -p /root/.ssh; echo "Host *" > /root/.ssh/config; \
+ echo "StrictHostKeyChecking no" >> /root/.ssh/config; \
+ echo "UserKnownHostsFile=/dev/null" >> /root/.ssh/config; \
+ echo "GlobalKnownHostsFile=/dev/null" >> /root/.ssh/config; \
+ echo "ConnectTimeout=30" >> /root/.ssh/config
+
+# Configuring log directories
+RUN mkdir -p /response; chmod a+rwx /response && \
+ mkdir -p /logs/ssn; chmod a+rwx /logs/ssn && \
+ mkdir -p /logs/project; chmod a+rwx /logs/project && \
+ mkdir -p /logs/edge; chmod a+rwx /logs/edge && \
+ mkdir -p /logs/notebook; chmod a+rwx /logs/notebook && \
+ mkdir -p /logs/dataengine; chmod a+rwx /logs/dataengine && \
+ mkdir -p /logs/dataengine-service; chmod a+rwx /logs/dataengine-service
+
+# Copying all base scripts to docker
+ENV PROVISION_CONFIG_DIR /root/conf/
+ENV KEYFILE_DIR /root/keys/
+ENV AWS_DIR /root/.aws
+
+RUN mkdir -p /root/conf && \
+ mkdir -p /root/scripts && \
+ mkdir -p /root/templates && \
+ mkdir -p /root/files && \
+ mkdir -p /usr/lib/python2.7/dlab && \
+ mkdir -p /root/keys/.ssh
+
+COPY ${SRC_PATH}base/ /root
+COPY ${SRC_PATH}odahu/ /root
+COPY ${SRC_PATH}general/conf/* /root/conf/
+COPY ${SRC_PATH}general/api/*.py /bin/
+COPY ${SRC_PATH}general/scripts/aws/common_* /root/scripts/
+COPY ${SRC_PATH}general/scripts/aws/odahu_* /root/scripts/
+COPY ${SRC_PATH}general/lib/aws/* /usr/lib/python2.7/dlab/
+COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python2.7/dlab/common_lib.py
+COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python2.7/dlab/fab.py
+COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
+COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
+COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
+COPY ${SRC_PATH}project/templates/locations/ /root/locations/
+
+RUN chmod a+x /root/*.py && \
+ chmod a+x /root/scripts/* && \
+ chmod a+x /bin/*.py
+
+ENTRYPOINT ["/root/entrypoint.py"]
diff --git a/infrastructure-provisioning/src/general/files/aws/odahu_description.json b/infrastructure-provisioning/src/general/files/aws/odahu_description.json
new file mode 100644
index 0000000..160a2ec
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/aws/odahu_description.json
@@ -0,0 +1,8 @@
+{
+ "template_name": "OdahuFlow cluster",
+ "description": "OdahuFlow cluster",
+ "environment_type": "computational",
+ "templates":
+ [
+ ]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/files/gcp/odahu_Dockerfile b/infrastructure-provisioning/src/general/files/gcp/odahu_Dockerfile
new file mode 100644
index 0000000..995092d
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/gcp/odahu_Dockerfile
@@ -0,0 +1,85 @@
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+FROM ODAHU_IMAGE
+ARG OS
+ARG SRC_PATH
+
+ENV PROFILE=/profile.json
+
+# Install any .deb dependecies
+RUN apt-get update && \
+ apt-get -y upgrade && \
+ apt-get -y install python-pip build-essential python3 python3-dev python-dev && \
+ apt-get -y install groff vim less git wget nano libssl-dev libffi-dev libffi6 && \
+ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install any python dependencies
+RUN pip2 install -UI pip==9.0.3 && \
+ pip2 install boto3 backoff fabric==1.14.0 fabvenv argparse ujson pycrypto google-api-python-client google-cloud-storage \
+ pyyaml google-auth-httplib2 oauth2client
+
+# Configuring ssh for user
+RUN mkdir -p /root/.ssh; echo "Host *" > /root/.ssh/config; \
+ echo "StrictHostKeyChecking no" >> /root/.ssh/config; \
+ echo "UserKnownHostsFile=/dev/null" >> /root/.ssh/config; \
+ echo "GlobalKnownHostsFile=/dev/null" >> /root/.ssh/config; \
+ echo "ConnectTimeout=30" >> /root/.ssh/config
+
+# Configuring log directories
+RUN mkdir -p /response; chmod a+rwx /response && \
+ mkdir -p /logs/ssn; chmod a+rwx /logs/ssn && \
+ mkdir -p /logs/project; chmod a+rwx /logs/project && \
+ mkdir -p /logs/edge; chmod a+rwx /logs/edge && \
+ mkdir -p /logs/notebook; chmod a+rwx /logs/notebook && \
+ mkdir -p /logs/dataengine; chmod a+rwx /logs/dataengine && \
+ mkdir -p /logs/dataengine-service; chmod a+rwx /logs/dataengine-service
+
+# Copying all base scripts to docker
+ENV PROVISION_CONFIG_DIR /root/conf/
+ENV KEYFILE_DIR /root/keys/
+
+RUN mkdir -p /root/conf && \
+ mkdir -p /root/scripts && \
+ mkdir -p /root/templates && \
+ mkdir -p /root/files && \
+ mkdir -p /usr/lib/python2.7/dlab && \
+ mkdir -p /root/keys/.ssh
+
+COPY ${SRC_PATH}base/ /root
+COPY ${SRC_PATH}odahu/ /root
+COPY ${SRC_PATH}general/conf/* /root/conf/
+COPY ${SRC_PATH}general/api/*.py /bin/
+COPY ${SRC_PATH}general/scripts/gcp/common_* /root/scripts/
+COPY ${SRC_PATH}general/scripts/gcp/odahu_* /root/scripts/
+COPY ${SRC_PATH}general/lib/gcp/* /usr/lib/python2.7/dlab/
+COPY ${SRC_PATH}general/lib/os/${OS}/common_lib.py /usr/lib/python2.7/dlab/common_lib.py
+COPY ${SRC_PATH}general/lib/os/fab.py /usr/lib/python2.7/dlab/fab.py
+COPY ${SRC_PATH}general/files/os/${OS}/sources.list /root/files/
+COPY ${SRC_PATH}general/files/os/ivysettings.xml /root/templates/
+COPY ${SRC_PATH}general/files/os/local_endpoint.json /root/files/
+COPY ${SRC_PATH}project/templates/locations/ /root/locations/
+
+RUN chmod a+x /root/*.py && \
+ chmod a+x /root/scripts/* && \
+ chmod a+x /bin/*.py
+
+ENTRYPOINT ["/root/entrypoint.py"]
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/files/gcp/odahu_description.json b/infrastructure-provisioning/src/general/files/gcp/odahu_description.json
new file mode 100644
index 0000000..160a2ec
--- /dev/null
+++ b/infrastructure-provisioning/src/general/files/gcp/odahu_description.json
@@ -0,0 +1,8 @@
+{
+ "template_name": "OdahuFlow cluster",
+ "description": "OdahuFlow cluster",
+ "environment_type": "computational",
+ "templates":
+ [
+ ]
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/files/gcp/ssn_policy.json b/infrastructure-provisioning/src/general/files/gcp/ssn_policy.json
index bd95d12..eb4ad12 100644
--- a/infrastructure-provisioning/src/general/files/gcp/ssn_policy.json
+++ b/infrastructure-provisioning/src/general/files/gcp/ssn_policy.json
@@ -16,5 +16,23 @@
"compute.images.get",
"compute.images.delete",
"compute.images.setLabels",
- "compute.images.list"
+ "compute.images.list",
+ "container.clusters.create",
+ "container.clusters.delete",
+ "container.clusters.get",
+ "container.clusters.update",
+ "container.operations.get",
+ "compute.routers.create",
+ "compute.routers.delete",
+ "compute.routers.get",
+ "compute.routers.update",
+ "container.serviceAccounts.create",
+ "container.serviceAccounts.get",
+ "container.serviceAccounts.delete",
+ "iam.serviceAccountKeys.create",
+ "iam.serviceAccountKeys.delete",
+ "iam.serviceAccountKeys.get",
+ "dns.resourceRecordSets.list",
+ "dns.managedZones.get",
+ "dns.managedZones.delete"
]
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/files/gcp/ssn_roles.json b/infrastructure-provisioning/src/general/files/gcp/ssn_roles.json
index fe310b6..3a72260 100644
--- a/infrastructure-provisioning/src/general/files/gcp/ssn_roles.json
+++ b/infrastructure-provisioning/src/general/files/gcp/ssn_roles.json
@@ -7,5 +7,7 @@
"iam.roleAdmin",
"compute.instanceAdmin",
"bigquery.dataViewer",
- "bigquery.jobUser"
+ "bigquery.jobUser",
+ "container.admin",
+ "dns.admin"
]
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
index 9519053..7be5328 100644
--- a/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
+++ b/infrastructure-provisioning/src/general/lib/aws/actions_lib.py
@@ -1034,6 +1034,9 @@
elif bucket_type == 'edge':
bucket_name = (os.environ['conf_service_base_name'] + '-' + "{}".format(scientist) + '-' +
os.environ['endpoint_name'] + '-bucket').lower().replace('_', '-')
+ elif bucket_type == 'odahu':
+ bucket_name = "{}-{}-tfstate".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
else:
bucket_name = (os.environ['conf_service_base_name']).lower().replace('_', '-')
for item in client.list_buckets().get('Buckets'):
diff --git a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
index cc16028..431a859 100644
--- a/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
+++ b/infrastructure-provisioning/src/general/lib/gcp/meta_lib.py
@@ -758,6 +758,28 @@
file=sys.stdout)}))
traceback.print_exc(file=sys.stdout)
+ def get_available_zones(self):
+ try:
+ request = self.service.regions().get(project=self.project, region=os.environ['gcp_region'])
+ gpu_zones = ['asia-east1-a', 'asia-east1-c', 'australia1-c', 'us-central1-c', 'us-central1-f', 'us-east1-b', 'us-east1-c', 'us-west1-a', 'us-west1-b', 'europe-west1-b', 'europe-west1-d', 'europe-west4-a']
+ zone_full = []
+ zone_list = []
+ response = request.execute()
+ for zone in response['zones']:
+ zone_full.append(str(zone.split('/')[-1]))
+ for zone in zone_full:
+ if zone in gpu_zones:
+ zone_list.append(str(zone.split('/')[-1]))
+ return zone_list
+ except Exception as err:
+ logging.info(
+ "Error with getting available zones: " + str(err) + "\n Traceback: " + traceback.print_exc(
+ file=sys.stdout) + "\n T")
+ append_result(str({"error": "Error with getting available zones",
+ "error_message": str(err) + "\n Traceback: " + traceback.print_exc(file=sys.stdout)}))
+ traceback.print_exc(file=sys.stdout)
+ return ''
+
def get_instance_private_ip_address(tag_name, instance_name):
try:
diff --git a/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py b/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
new file mode 100644
index 0000000..e7e0363
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_deploy.py
@@ -0,0 +1,308 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import base64
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['aws_region'])
+ odahu_conf['zone'] = (os.environ['aws_zone'])
+ odahu_conf['edge_user_name'] = os.environ['edge_user_name']
+ if os.environ['aws_region'] == 'us-east':
+ odahu_conf['az_list'] = ["us-east-1", "us-east-2"]
+ elif os.environ['aws_region'] == 'ap-south':
+ odahu_conf['az_list'] = ["ap-south-1"]
+ elif os.environ['aws_region'] == 'us-west':
+ odahu_conf['az_list'] = ["us-west-1", "us-west-2"]
+ elif os.environ['aws_region'] == 'eu-west':
+ odahu_conf['az_list'] = ["eu-west-1", "eu-west-2", "eu-west-3"]
+ elif os.environ['aws_region'] == 'ca-central':
+ odahu_conf['az_list'] = ["ca-central-1"]
+ odahu_conf['az_list'] = GCPMeta().get_available_zones()
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+ odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
+ odahu_conf['grafana_pass'] = id_generator()
+ odahu_conf['docker_password'] = base64.b64decode(os.environ['odahu_docker_password'] + "==")
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['keycloak_auth_server_url'].replace('https://', '').replace('/auth', '')
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = base64.b64encode(id_generator(16))
+ odahu_conf['oauth_local_jwks'] = os.environ['odahu_oauth_local_jwks']
+ odahu_conf['infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['private_subnet_cidrs'] = os.environ['odahu_private_subnet_cidrs'].split(',')
+ odahu_conf['public_subnet_cidrs'] = os.environ['odahu_public_subnet_cidrs'].split(',')
+ odahu_conf['infra_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = id_generator()
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ odahu_conf['automation_version'] = os.environ['odahu_automation_version']
+ odahu_conf['ui_version'] = os.environ['odahu_ui_version']
+ odahu_conf['examples_version'] = os.environ['odahu_examples_version']
+ odahu_conf['jupyterhub_enabled'] = os.environ['odahu_jupyterhub_enabled']
+ odahu_conf['oauth_mesh_enabled'] = os.environ['odahu_oauth_mesh_enabled']
+ odahu_conf['keysecret'] = os.environ['odahu_keysecret']
+ odahu_conf['airflow_secret'] = os.environ['odahu_airflow_secret']
+ odahu_conf['operator_secret'] =os.environ['odahu_operator_secret']
+ odahu_conf['resource-uploader_secret'] = os.environ['odahu_resource_uploader_secret']
+ odahu_conf['tester_secret'] = os.environ['odahu_tester_secret']
+ odahu_conf['tester-data-scientist_secret'] = os.environ['odahu_tester_data_scientist_secret']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof ={
+ "allowed_ips": odahu_conf['allowed_cidr'],
+ "authorization_enabled": "true",
+ "authz_dry_run": "false",
+ "cloud": {
+ "aws": {
+ "az_list": odahu_conf['az_list'],
+ "zone": "{}".format(odahu_conf['zone']),
+ },
+ "region": "{}".format(odahu_conf['region']),
+ "type": "aws"
+ },
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "aws/eks",
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+ "dns": {
+ "domain": "odahu.{}.{}".format(odahu_conf['cluster_name'], odahu_conf['root_domain']),
+ "gcp_credentials": "{}",
+ "gcp_project_id": "{}".format(odahu_conf['project_id']),
+ "provider": "gcp",
+ "zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ },
+ "docker_password": "{}".format(odahu_conf['docker_password']),
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_username": "_json_key",
+ "infra_cidr": "{}".format(odahu_conf['infra_cidr']),
+ "examples_version": "{}".format(odahu_conf['examples_version']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "jupyterhub_enabled": odahu_conf['jupyterhub_enabled'],
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "k8s_version": "{}".format(odahu_conf['k8s_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "nat_subnet_cidr": "{}".format(odahu_conf['cidr']),
+ "node_pools": {
+ "main": {
+ "init_node_count": 1,
+ "max_node_count": 7,
+ "min_node_count": 1
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "deployment"
+ }
+ ]
+ },
+ "packaging": {
+ "disk_size_gb": 100,
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "packaging"
+ }
+ ]
+ },
+ "training": {
+ "disk_size_gb": 50,
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "machine_type": "c5.2xlarge",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training"
+ }
+ ]
+ }
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_local_jwks": "{}".format(odahu_conf['oauth_local_jwks']),
+ "oauth_mesh_enabled": odahu_conf['oauth_mesh_enabled'],
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_jwks_url": "{}/protocol/openid-connect/certs".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_port": 443,
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_token_endpoint": "{}/protocol/openid-connect/token".format(odahu_conf['oauth_oidc_issuer_url']),
+ "odahu_automation_version": "{}".format(odahu_conf['automation_version']),
+ "odahu_infra_version": "{}".format(odahu_conf['infra_version']),
+ "odahu_ui_version": "{}".format(odahu_conf['ui_version']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "odahuflow_connections": [
+ {
+ "id": "odahu-flow-examples",
+ "spec": {
+ "description": "Git repository with the Odahu-Flow examples",
+ "keySecret": "{}".format(odahu_conf['keysecret']),
+ "reference": "{}".format(odahu_conf['examples_version']),
+ "type": "git",
+ "uri": "git@github.com:odahu/odahu-examples.git",
+ "webUILink": "https://github.com/odahu/odahu-examples"
+ }
+ }
+ ],
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "opa_policies": {},
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "private_subnet_cidrs": odahu_conf['private_subnet_cidrs'],
+ "public_subnet_cidrs": odahu_conf['public_subnet_cidrs'],
+ "service_accounts": {
+ "airflow": {
+ "client_id": "sa-airflow",
+ "client_secret": "{}".format(odahu_conf['airflow_secret'])
+ },
+ "operator": {
+ "client_id": "sa-operator",
+ "client_secret": "{}".format(odahu_conf['operator_secret'])
+ },
+ "resource_uploader": {
+ "client_id": "sa-resource-uploader",
+ "client_secret": "{}".format(odahu_conf['resource-uploader_secret'])
+ },
+ "test": {
+ "client_id": "sa-tester",
+ "client_secret": "{}".format(odahu_conf['tester_secret'])
+ },
+ "test_data_scientist": {
+ "client_id": "sa-tester-data-scientist",
+ "client_secret": "{}".format(odahu_conf['tester-data-scientist_secret'])
+ }
+ },
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "vault": {
+ "enabled": "true"
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
+ local('cp /tmp/profile.json /')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ GCPActions().remove_bucket(odahu_conf['bucket_name'])
+ GCPActions().remove_static_address(odahu_conf['static_address_name'], odahu_conf['region'])
+ sys.exit(1)
+
+ try:
+ local('tf_runner create -o /tmp/result.json')
+ local("sed -i 's|name|description|g' /tmp/result.json")
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to deploy Odahu cluster.", str(err))
+ GCPActions().remove_bucket(odahu_conf['bucket_name'])
+ GCPActions().remove_static_address(odahu_conf['static_address_name'], odahu_conf['region'])
+ sys.exit(1)
+
+ # generating output information
+ print('[SUMMARY]')
+ logging.info('[SUMMARY]')
+ print('Cluster name: {}'.format(odahu_conf['cluster_name']))
+ with open('/tmp/result.json', 'r') as f:
+ output = json.load(f)
+ odahu_urls = json.dumps(output['odahu_urls']['value'], sort_keys=True, indent=4)
+ print('Odahu urls: {}'.format(odahu_urls))
+ res = dict()
+ res['odahu_urls'] = output['odahu_urls']['value']
+ res['oauth_cookie_secret'] = odahu_conf['oauth_cookie_secret']
+ res['odahuflow_connection_decrypt_token'] = odahu_conf['decrypt_token']
+ res['grafana_pass'] = odahu_conf['grafana_pass']
+ res['grafana_admin'] = odahu_conf['grafana_admin']
+ with open("/root/result.json", 'w') as result:
+ result.write(json.dumps(res))
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py b/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
new file mode 100644
index 0000000..d4168ed
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_prepare.py
@@ -0,0 +1,155 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+import requests
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['endpoint_name'] = (os.environ['endpoint_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['tag_name'] = '{}-tag'.format(odahu_conf['service_base_name'])
+ odahu_conf['endpoint_tag'] = (os.environ['endpoint_name']).lower().replace('_', '-')
+ odahu_conf['project_tag'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['region'] = os.environ['gcp_region']
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ odahu_conf['keycloak_auth_server_url'] = os.environ['keycloak_auth_server_url']
+ odahu_conf['keycloak_realm_name'] = os.environ['keycloak_realm_name']
+ odahu_conf['keycloak_client_name'] = os.environ['keycloak_client_name']
+ odahu_conf['keycloak_user'] = os.environ['keycloak_user']
+ odahu_conf['keycloak_user_password'] = os.environ['keycloak_user_password']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ if 'conf_additional_tags' in os.environ:
+ odahu_conf['bucket_additional_tags'] = ';' + os.environ['conf_additional_tags']
+ os.environ['conf_additional_tags'] = os.environ['conf_additional_tags'] + \
+ ';project_tag:{0};endpoint_tag:{1};'.format(
+ odahu_conf['project_tag'], odahu_conf['endpoint_tag'])
+ else:
+ odahu_conf['bucket_additional_tags'] = ''
+ os.environ['conf_additional_tags'] = 'project_tag:{0};endpoint_tag:{1}'.format(odahu_conf['project_tag'],
+ odahu_conf['endpoint_tag'])
+ print('Additional tags will be added: {}'.format(os.environ['conf_additional_tags']))
+
+
+ try:
+ logging.info('[CREATE STATE BUCKETS]')
+ print('[CREATE STATE BUCKETS]')
+
+ odahu_conf['bucket_tags'] = 'endpoint_tag:{0};{1}:{2};project_tag:{3};{4}:{5}{6}'\
+ .format(odahu_conf['endpoint_tag'], os.environ['conf_billing_tag_key'], os.environ['conf_billing_tag_value'],
+ odahu_conf['project_tag'], odahu_conf['tag_name'], odahu_conf['bucket_name'],
+ odahu_conf['bucket_additional_tags']).replace(';', ',')
+
+ params = "--bucket_name {0} --bucket_tags {1} --region {2} --bucket_name_tag {0}". \
+ format(odahu_conf['bucket_name'], odahu_conf['bucket_tags'], odahu_conf['region'])
+ try:
+ local("~/scripts/{}.py {}".format('common_create_bucket', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Unable to create bucket.", str(err))
+ sys.exit(1)
+
+ try:
+ logging.info('[CREATE NAT GATEWAY]')
+ print('[CREATE NAT GATEWAY]')
+ print("Allocating Elastic IP")
+ allocation_id = allocate_elastic_ip()
+ tag = {"Key": odahu_conf['tag_name'], "Value": odahu_conf['static_address_name']}
+ tag_name = {"Key": "Name", "Value": odahu_conf['static_address_name']}
+ create_tag(allocation_id, tag)
+ create_tag(allocation_id, tag_name)
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Unable to reserve static ip.", str(err))
+ remove_s3(bucket_type='odahu')
+ sys.exit(1)
+
+ try:
+ print('[CONFIGURE REDIRECT URI]')
+ logging.info('[CONFIGURE REDIRECT URI]')
+ keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format(
+ odahu_conf['keycloak_auth_server_url'])
+ keycloak_auth_data = {
+ "username": odahu_conf['keycloak_user'],
+ "password": odahu_conf['keycloak_user_password'],
+ "grant_type": "password",
+ "client_id": "admin-cli",
+ }
+ keycloak_client_create_url = '{0}/admin/realms/{1}/clients'.format(odahu_conf['keycloak_auth_server_url'],
+ odahu_conf['keycloak_realm_name'])
+ odahu_redirectUris = 'https://odahu.{0}.{1}/*,http://odahu.{0}.{1}/*'.format(odahu_conf['cluster_name'],
+ odahu_conf['root_domain']).split(',')
+
+
+ try:
+ keycloak_token = requests.post(keycloak_auth_server_url, data=keycloak_auth_data, verify=False).json()
+ keycloak_get_Uris = requests.get(keycloak_client_create_url,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"}, verify=False).json()
+ for dict in keycloak_get_Uris:
+ if dict["clientId"] == odahu_conf['keycloak_client_name']:
+ ui_redirectUris = dict["redirectUris"]
+ keycloak_client_id = dict["id"]
+ keycloak_redirectUris = odahu_redirectUris + ui_redirectUris
+ updated_client_data = {
+ "clientId": odahu_conf['keycloak_client_name'],
+ "id": keycloak_client_id,
+ "enabled": "true",
+ "redirectUris": keycloak_redirectUris,
+ "publicClient": "false",
+ "protocol": "openid-connect",
+ }
+ client_url = "{}/{}".format(keycloak_client_create_url, keycloak_client_id)
+ keycloak_update_Uris = requests.put(client_url, json=updated_client_data,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"}, verify=False)
+ except Exception as err:
+ append_result("Failed to configure keycloak.")
+ sys.exit(1)
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure keycloak.", str(err))
+ remove_s3(bucket_type='odahu')
+ release_elastic_ip(allocation_id)
+ sys.exit(1)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/aws/odahu_resume.py b/infrastructure-provisioning/src/general/scripts/aws/odahu_resume.py
new file mode 100644
index 0000000..245336a
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_resume.py
@@ -0,0 +1,289 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import string
+import random
+import base64
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['aws_region'])
+ odahu_conf['zone'] = (os.environ['aws_zone'])
+ odahu_conf['edge_user_name'] = os.environ['edge_user_name']
+ if os.environ['aws_region'] == 'us-east':
+ odahu_conf['az_list'] = ["us-east-1", "us-east-2"]
+ elif os.environ['aws_region'] == 'ap-south':
+ odahu_conf['az_list'] = ["ap-south-1"]
+ elif os.environ['aws_region'] == 'us-west':
+ odahu_conf['az_list'] = ["us-west-1", "us-west-2"]
+ elif os.environ['aws_region'] == 'eu-west':
+ odahu_conf['az_list'] = ["eu-west-1", "eu-west-2", "eu-west-3"]
+ elif os.environ['aws_region'] == 'ca-central':
+ odahu_conf['az_list'] = ["ca-central-1"]
+ odahu_conf['az_list'] = GCPMeta().get_available_zones()
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+ odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['grafana_pass']
+ odahu_conf['docker_password'] = base64.b64decode(os.environ['odahu_docker_password'] + "==")
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['keycloak_auth_server_url'].replace('https://', '').replace('/auth', '')
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = os.environ['oauth_cookie_secret']
+ odahu_conf['oauth_local_jwks'] = os.environ['odahu_oauth_local_jwks']
+ odahu_conf['infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['private_subnet_cidrs'] = os.environ['odahu_private_subnet_cidrs'].split(',')
+ odahu_conf['public_subnet_cidrs'] = os.environ['odahu_public_subnet_cidrs'].split(',')
+ odahu_conf['infra_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = os.environ['decrypt_token']
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ odahu_conf['automation_version'] = os.environ['odahu_automation_version']
+ odahu_conf['ui_version'] = os.environ['odahu_ui_version']
+ odahu_conf['examples_version'] = os.environ['odahu_examples_version']
+ odahu_conf['jupyterhub_enabled'] = os.environ['odahu_jupyterhub_enabled']
+ odahu_conf['oauth_mesh_enabled'] = os.environ['odahu_oauth_mesh_enabled']
+ odahu_conf['keysecret'] = os.environ['odahu_keysecret']
+ odahu_conf['airflow_secret'] = os.environ['odahu_airflow_secret']
+ odahu_conf['operator_secret'] = os.environ['odahu_operator_secret']
+ odahu_conf['resource-uploader_secret'] = os.environ['odahu_resource_uploader_secret']
+ odahu_conf['tester_secret'] = os.environ['odahu_tester_secret']
+ odahu_conf['tester-data-scientist_secret'] = os.environ['odahu_tester_data_scientist_secret']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {
+ "allowed_ips": odahu_conf['allowed_cidr'],
+ "authorization_enabled": "true",
+ "authz_dry_run": "false",
+ "cloud": {
+ "aws": {
+ "az_list": odahu_conf['az_list'],
+ "zone": "{}".format(odahu_conf['zone']),
+ },
+ "region": "{}".format(odahu_conf['region']),
+ "type": "aws"
+ },
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "aws/eks",
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+ "dns": {
+ "domain": "odahu.{}.{}".format(odahu_conf['cluster_name'], odahu_conf['root_domain']),
+ "gcp_credentials": "{}",
+ "gcp_project_id": "{}".format(odahu_conf['project_id']),
+ "provider": "gcp",
+ "zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ },
+ "docker_password": "{}".format(odahu_conf['docker_password']),
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_username": "_json_key",
+ "infra_cidr": "{}".format(odahu_conf['infra_cidr']),
+ "examples_version": "{}".format(odahu_conf['examples_version']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "jupyterhub_enabled": odahu_conf['jupyterhub_enabled'],
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "k8s_version": "{}".format(odahu_conf['k8s_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "nat_subnet_cidr": "{}".format(odahu_conf['cidr']),
+ "node_pools": {
+ "main": {
+ "init_node_count": 1,
+ "max_node_count": 7,
+ "min_node_count": 1
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "deployment"
+ }
+ ]
+ },
+ "packaging": {
+ "disk_size_gb": 100,
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "packaging"
+ }
+ ]
+ },
+ "training": {
+ "disk_size_gb": 50,
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "machine_type": "c5.2xlarge",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training"
+ }
+ ]
+ }
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_local_jwks": "{}".format(odahu_conf['oauth_local_jwks']),
+ "oauth_mesh_enabled": odahu_conf['oauth_mesh_enabled'],
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_jwks_url": "{}/protocol/openid-connect/certs".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_port": 443,
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_token_endpoint": "{}/protocol/openid-connect/token".format(
+ odahu_conf['oauth_oidc_issuer_url']),
+ "odahu_automation_version": "{}".format(odahu_conf['automation_version']),
+ "odahu_infra_version": "{}".format(odahu_conf['infra_version']),
+ "odahu_ui_version": "{}".format(odahu_conf['ui_version']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "odahuflow_connections": [
+ {
+ "id": "odahu-flow-examples",
+ "spec": {
+ "description": "Git repository with the Odahu-Flow examples",
+ "keySecret": "{}".format(odahu_conf['keysecret']),
+ "reference": "{}".format(odahu_conf['examples_version']),
+ "type": "git",
+ "uri": "git@github.com:odahu/odahu-examples.git",
+ "webUILink": "https://github.com/odahu/odahu-examples"
+ }
+ }
+ ],
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "opa_policies": {},
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "private_subnet_cidrs": odahu_conf['private_subnet_cidrs'],
+ "public_subnet_cidrs": odahu_conf['public_subnet_cidrs'],
+ "service_accounts": {
+ "airflow": {
+ "client_id": "sa-airflow",
+ "client_secret": "{}".format(odahu_conf['airflow_secret'])
+ },
+ "operator": {
+ "client_id": "sa-operator",
+ "client_secret": "{}".format(odahu_conf['operator_secret'])
+ },
+ "resource_uploader": {
+ "client_id": "sa-resource-uploader",
+ "client_secret": "{}".format(odahu_conf['resource-uploader_secret'])
+ },
+ "test": {
+ "client_id": "sa-tester",
+ "client_secret": "{}".format(odahu_conf['tester_secret'])
+ },
+ "test_data_scientist": {
+ "client_id": "sa-tester-data-scientist",
+ "client_secret": "{}".format(odahu_conf['tester-data-scientist_secret'])
+ }
+ },
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "vault": {
+ "enabled": "true"
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
+ local('cp /tmp/profile.json /')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
+
+ try:
+ local('tf_runner resume -v')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to deploy Odahu cluster.", str(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/odahu_suspend.py b/infrastructure-provisioning/src/general/scripts/aws/odahu_suspend.py
new file mode 100644
index 0000000..7a1faa8
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_suspend.py
@@ -0,0 +1,287 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import base64
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['aws_region'])
+ odahu_conf['zone'] = (os.environ['aws_zone'])
+ odahu_conf['edge_user_name'] = os.environ['edge_user_name']
+ if os.environ['aws_region'] == 'us-east':
+ odahu_conf['az_list'] = ["us-east-1", "us-east-2"]
+ elif os.environ['aws_region'] == 'ap-south':
+ odahu_conf['az_list'] = ["ap-south-1"]
+ elif os.environ['aws_region'] == 'us-west':
+ odahu_conf['az_list'] = ["us-west-1", "us-west-2"]
+ elif os.environ['aws_region'] == 'eu-west':
+ odahu_conf['az_list'] = ["eu-west-1", "eu-west-2", "eu-west-3"]
+ elif os.environ['aws_region'] == 'ca-central':
+ odahu_conf['az_list'] = ["ca-central-1"]
+ odahu_conf['az_list'] = GCPMeta().get_available_zones()
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+ odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['grafana_pass']
+ odahu_conf['docker_password'] = base64.b64decode(os.environ['odahu_docker_password'] + "==")
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['keycloak_auth_server_url'].replace('https://', '').replace('/auth', '')
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = os.environ['oauth_cookie_secret']
+ odahu_conf['oauth_local_jwks'] = os.environ['odahu_oauth_local_jwks']
+ odahu_conf['infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['private_subnet_cidrs'] = os.environ['odahu_private_subnet_cidrs'].split(',')
+ odahu_conf['public_subnet_cidrs'] = os.environ['odahu_public_subnet_cidrs'].split(',')
+ odahu_conf['infra_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = os.environ['decrypt_token']
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ odahu_conf['automation_version'] = os.environ['odahu_automation_version']
+ odahu_conf['ui_version'] = os.environ['odahu_ui_version']
+ odahu_conf['examples_version'] = os.environ['odahu_examples_version']
+ odahu_conf['jupyterhub_enabled'] = os.environ['odahu_jupyterhub_enabled']
+ odahu_conf['oauth_mesh_enabled'] = os.environ['odahu_oauth_mesh_enabled']
+ odahu_conf['keysecret'] = os.environ['odahu_keysecret']
+ odahu_conf['airflow_secret'] = os.environ['odahu_airflow_secret']
+ odahu_conf['operator_secret'] = os.environ['odahu_operator_secret']
+ odahu_conf['resource-uploader_secret'] = os.environ['odahu_resource_uploader_secret']
+ odahu_conf['tester_secret'] = os.environ['odahu_tester_secret']
+ odahu_conf['tester-data-scientist_secret'] = os.environ['odahu_tester_data_scientist_secret']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {
+ "allowed_ips": odahu_conf['allowed_cidr'],
+ "authorization_enabled": "true",
+ "authz_dry_run": "false",
+ "cloud": {
+ "aws": {
+ "az_list": odahu_conf['az_list'],
+ "zone": "{}".format(odahu_conf['zone']),
+ },
+ "region": "{}".format(odahu_conf['region']),
+ "type": "aws"
+ },
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "aws/eks",
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+ "dns": {
+ "domain": "odahu.{}.{}".format(odahu_conf['cluster_name'], odahu_conf['root_domain']),
+ "gcp_credentials": "{}",
+ "gcp_project_id": "{}".format(odahu_conf['project_id']),
+ "provider": "gcp",
+ "zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ },
+ "docker_password": "{}".format(odahu_conf['docker_password']),
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_username": "_json_key",
+ "infra_cidr": "{}".format(odahu_conf['infra_cidr']),
+ "examples_version": "{}".format(odahu_conf['examples_version']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "jupyterhub_enabled": odahu_conf['jupyterhub_enabled'],
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "k8s_version": "{}".format(odahu_conf['k8s_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "nat_subnet_cidr": "{}".format(odahu_conf['cidr']),
+ "node_pools": {
+ "main": {
+ "init_node_count": 1,
+ "max_node_count": 7,
+ "min_node_count": 1
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "deployment"
+ }
+ ]
+ },
+ "packaging": {
+ "disk_size_gb": 100,
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "packaging"
+ }
+ ]
+ },
+ "training": {
+ "disk_size_gb": 50,
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "machine_type": "c5.2xlarge",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training"
+ }
+ ]
+ }
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_local_jwks": "{}".format(odahu_conf['oauth_local_jwks']),
+ "oauth_mesh_enabled": odahu_conf['oauth_mesh_enabled'],
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_jwks_url": "{}/protocol/openid-connect/certs".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_port": 443,
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_token_endpoint": "{}/protocol/openid-connect/token".format(
+ odahu_conf['oauth_oidc_issuer_url']),
+ "odahu_automation_version": "{}".format(odahu_conf['automation_version']),
+ "odahu_infra_version": "{}".format(odahu_conf['infra_version']),
+ "odahu_ui_version": "{}".format(odahu_conf['ui_version']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "odahuflow_connections": [
+ {
+ "id": "odahu-flow-examples",
+ "spec": {
+ "description": "Git repository with the Odahu-Flow examples",
+ "keySecret": "{}".format(odahu_conf['keysecret']),
+ "reference": "{}".format(odahu_conf['examples_version']),
+ "type": "git",
+ "uri": "git@github.com:odahu/odahu-examples.git",
+ "webUILink": "https://github.com/odahu/odahu-examples"
+ }
+ }
+ ],
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "opa_policies": {},
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "private_subnet_cidrs": odahu_conf['private_subnet_cidrs'],
+ "public_subnet_cidrs": odahu_conf['public_subnet_cidrs'],
+ "service_accounts": {
+ "airflow": {
+ "client_id": "sa-airflow",
+ "client_secret": "{}".format(odahu_conf['airflow_secret'])
+ },
+ "operator": {
+ "client_id": "sa-operator",
+ "client_secret": "{}".format(odahu_conf['operator_secret'])
+ },
+ "resource_uploader": {
+ "client_id": "sa-resource-uploader",
+ "client_secret": "{}".format(odahu_conf['resource-uploader_secret'])
+ },
+ "test": {
+ "client_id": "sa-tester",
+ "client_secret": "{}".format(odahu_conf['tester_secret'])
+ },
+ "test_data_scientist": {
+ "client_id": "sa-tester-data-scientist",
+ "client_secret": "{}".format(odahu_conf['tester-data-scientist_secret'])
+ }
+ },
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "vault": {
+ "enabled": "true"
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
+ local('cp /tmp/profile.json /')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
+
+ try:
+ local('tf_runner suspend -v')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to suspend Odahu cluster.", str(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/odahu_terminate.py b/infrastructure-provisioning/src/general/scripts/aws/odahu_terminate.py
new file mode 100644
index 0000000..20f5468
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/aws/odahu_terminate.py
@@ -0,0 +1,307 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import base64
+
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['aws_region'])
+ odahu_conf['zone'] = (os.environ['aws_zone'])
+ odahu_conf['edge_user_name'] = os.environ['edge_user_name']
+ if os.environ['aws_region'] == 'us-east':
+ odahu_conf['az_list'] = ["us-east-1", "us-east-2"]
+ elif os.environ['aws_region'] == 'ap-south':
+ odahu_conf['az_list'] = ["ap-south-1"]
+ elif os.environ['aws_region'] == 'us-west':
+ odahu_conf['az_list'] = ["us-west-1", "us-west-2"]
+ elif os.environ['aws_region'] == 'eu-west':
+ odahu_conf['az_list'] = ["eu-west-1", "eu-west-2", "eu-west-3"]
+ elif os.environ['aws_region'] == 'ca-central':
+ odahu_conf['az_list'] = ["ca-central-1"]
+ odahu_conf['az_list'] = GCPMeta().get_available_zones()
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+ odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['grafana_pass']
+ odahu_conf['docker_password'] = base64.b64decode(os.environ['odahu_docker_password'] + "==")
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['keycloak_auth_server_url'].replace('https://', '').replace('/auth', '')
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = os.environ['oauth_cookie_secret']
+ odahu_conf['oauth_local_jwks'] = os.environ['odahu_oauth_local_jwks']
+ odahu_conf['infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['private_subnet_cidrs'] = os.environ['odahu_private_subnet_cidrs'].split(',')
+ odahu_conf['public_subnet_cidrs'] = os.environ['odahu_public_subnet_cidrs'].split(',')
+ odahu_conf['infra_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = os.environ['decrypt_token']
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ odahu_conf['automation_version'] = os.environ['odahu_automation_version']
+ odahu_conf['ui_version'] = os.environ['odahu_ui_version']
+ odahu_conf['examples_version'] = os.environ['odahu_examples_version']
+ odahu_conf['jupyterhub_enabled'] = os.environ['odahu_jupyterhub_enabled']
+ odahu_conf['oauth_mesh_enabled'] = os.environ['odahu_oauth_mesh_enabled']
+ odahu_conf['keysecret'] = os.environ['odahu_keysecret']
+ odahu_conf['airflow_secret'] = os.environ['odahu_airflow_secret']
+ odahu_conf['operator_secret'] = os.environ['odahu_operator_secret']
+ odahu_conf['resource-uploader_secret'] = os.environ['odahu_resource_uploader_secret']
+ odahu_conf['tester_secret'] = os.environ['odahu_tester_secret']
+ odahu_conf['tester-data-scientist_secret'] = os.environ['odahu_tester_data_scientist_secret']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {
+ "allowed_ips": odahu_conf['allowed_cidr'],
+ "authorization_enabled": "true",
+ "authz_dry_run": "false",
+ "cloud": {
+ "aws": {
+ "az_list": odahu_conf['az_list'],
+ "zone": "{}".format(odahu_conf['zone']),
+ },
+ "region": "{}".format(odahu_conf['region']),
+ "type": "aws"
+ },
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "aws/eks",
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+ "dns": {
+ "domain": "odahu.{}.{}".format(odahu_conf['cluster_name'], odahu_conf['root_domain']),
+ "gcp_credentials": "{}",
+ "gcp_project_id": "{}".format(odahu_conf['project_id']),
+ "provider": "gcp",
+ "zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ },
+ "docker_password": "{}".format(odahu_conf['docker_password']),
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_username": "_json_key",
+ "infra_cidr": "{}".format(odahu_conf['infra_cidr']),
+ "examples_version": "{}".format(odahu_conf['examples_version']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "jupyterhub_enabled": odahu_conf['jupyterhub_enabled'],
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "k8s_version": "{}".format(odahu_conf['k8s_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "nat_subnet_cidr": "{}".format(odahu_conf['cidr']),
+ "node_pools": {
+ "main": {
+ "init_node_count": 1,
+ "max_node_count": 7,
+ "min_node_count": 1
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "deployment"
+ }
+ ]
+ },
+ "packaging": {
+ "disk_size_gb": 100,
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "packaging"
+ }
+ ]
+ },
+ "training": {
+ "disk_size_gb": 50,
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "machine_type": "c5.2xlarge",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training"
+ }
+ ]
+ }
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_local_jwks": "{}".format(odahu_conf['oauth_local_jwks']),
+ "oauth_mesh_enabled": odahu_conf['oauth_mesh_enabled'],
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_jwks_url": "{}/protocol/openid-connect/certs".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_port": 443,
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_token_endpoint": "{}/protocol/openid-connect/token".format(
+ odahu_conf['oauth_oidc_issuer_url']),
+ "odahu_automation_version": "{}".format(odahu_conf['automation_version']),
+ "odahu_infra_version": "{}".format(odahu_conf['infra_version']),
+ "odahu_ui_version": "{}".format(odahu_conf['ui_version']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "odahuflow_connections": [
+ {
+ "id": "odahu-flow-examples",
+ "spec": {
+ "description": "Git repository with the Odahu-Flow examples",
+ "keySecret": "{}".format(odahu_conf['keysecret']),
+ "reference": "{}".format(odahu_conf['examples_version']),
+ "type": "git",
+ "uri": "git@github.com:odahu/odahu-examples.git",
+ "webUILink": "https://github.com/odahu/odahu-examples"
+ }
+ }
+ ],
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "opa_policies": {},
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "private_subnet_cidrs": odahu_conf['private_subnet_cidrs'],
+ "public_subnet_cidrs": odahu_conf['public_subnet_cidrs'],
+ "service_accounts": {
+ "airflow": {
+ "client_id": "sa-airflow",
+ "client_secret": "{}".format(odahu_conf['airflow_secret'])
+ },
+ "operator": {
+ "client_id": "sa-operator",
+ "client_secret": "{}".format(odahu_conf['operator_secret'])
+ },
+ "resource_uploader": {
+ "client_id": "sa-resource-uploader",
+ "client_secret": "{}".format(odahu_conf['resource-uploader_secret'])
+ },
+ "test": {
+ "client_id": "sa-tester",
+ "client_secret": "{}".format(odahu_conf['tester_secret'])
+ },
+ "test_data_scientist": {
+ "client_id": "sa-tester-data-scientist",
+ "client_secret": "{}".format(odahu_conf['tester-data-scientist_secret'])
+ }
+ },
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "vault": {
+ "enabled": "true"
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
+ local('cp /tmp/profile.json /')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
+
+ print('Removing Odahu cluster')
+ try:
+ local('tf_runner destroy')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to terminate Odahu cluster.", str(err))
+ sys.exit(1)
+
+ try:
+ buckets = GCPMeta().get_list_buckets(odahu_conf['cluster_name'])
+ if 'items' in buckets:
+ for i in buckets['items']:
+ GCPActions().remove_bucket(i['name'])
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ sys.exit(1)
+
+ try:
+ static_addresses = GCPMeta().get_list_static_addresses(odahu_conf['region'], odahu_conf['cluster_name'])
+ if 'items' in static_addresses:
+ for i in static_addresses['items']:
+ GCPActions().remove_static_address(i['name'], odahu_conf['region'])
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
index 87587d8..64fd85b 100644
--- a/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/aws/ssn_configure.py
@@ -243,6 +243,7 @@
additional_config = [{"name": "base", "tag": "latest"},
{"name": "edge", "tag": "latest"},
{"name": "project", "tag": "latest"},
+ {"name": "odahu", "tag": "latest"},
{"name": "jupyter", "tag": "latest"},
{"name": "rstudio", "tag": "latest"},
{"name": "zeppelin", "tag": "latest"},
@@ -253,13 +254,13 @@
{"name": "dataengine-service", "tag": "latest"},
{"name": "dataengine", "tag": "latest"}]
params = "--hostname {} --keyfile {} --additional_config '{}' --os_family {} --os_user {} --dlab_path {} " \
- "--cloud_provider {} --region {}".format(ssn_conf['instance_hostname'],
+ "--cloud_provider {} --region {} --gcr_creds {} --odahu_image {}".format(ssn_conf['instance_hostname'],
"{}{}.pem".format(os.environ['conf_key_dir'],
os.environ['conf_key_name']),
json.dumps(additional_config), os.environ['conf_os_family'],
ssn_conf['dlab_ssh_user'], os.environ['ssn_dlab_path'],
- os.environ['conf_cloud_provider'], os.environ['aws_region'])
-
+ os.environ['conf_cloud_provider'], os.environ['aws_region'],
+ os.environ['ssn_gcr_creds'], os.environ['odahu_deploy_image'])
try:
local("~/scripts/{}.py {}".format('configure_docker', params))
except:
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
new file mode 100644
index 0000000..0277d43
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_deploy.py
@@ -0,0 +1,319 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import base64
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['gcp_region'])
+ odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['edge_user_name'] = os.environ['edge_user_name']
+ odahu_conf['node_locations'] = GCPMeta().get_available_zones()
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+ odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['odahu_grafana_admin']
+ odahu_conf['grafana_pass'] = id_generator()
+ odahu_conf['docker_password'] = base64.b64decode(os.environ['odahu_docker_password'] + "==")
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['keycloak_auth_server_url'].replace('https://', '').replace('/auth', '')
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = base64.b64encode(id_generator(16))
+ odahu_conf['oauth_local_jwks'] = os.environ['odahu_oauth_local_jwks']
+ odahu_conf['infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['node_version'] = os.environ['odahu_node_version']
+ odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = id_generator()
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ odahu_conf['automation_version'] = os.environ['odahu_automation_version']
+ odahu_conf['ui_version'] = os.environ['odahu_ui_version']
+ odahu_conf['examples_version'] = os.environ['odahu_examples_version']
+ odahu_conf['jupyterhub_enabled'] = os.environ['odahu_jupyterhub_enabled']
+ odahu_conf['oauth_mesh_enabled'] = os.environ['odahu_oauth_mesh_enabled']
+ odahu_conf['keysecret'] = os.environ['odahu_keysecret']
+ odahu_conf['airflow_secret'] = os.environ['odahu_airflow_secret']
+ odahu_conf['operator_secret'] =os.environ['odahu_operator_secret']
+ odahu_conf['resource-uploader_secret'] = os.environ['odahu_resource_uploader_secret']
+ odahu_conf['tester_secret'] = os.environ['odahu_tester_secret']
+ odahu_conf['tester-data-scientist_secret'] = os.environ['odahu_tester_data_scientist_secret']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof ={
+ "allowed_ips": odahu_conf['allowed_cidr'],
+ "authorization_enabled": "true",
+ "authz_dry_run": "false",
+ "cloud": {
+ "gcp": {
+ "node_locations": odahu_conf['node_locations'],
+ "project_id": "{}".format(odahu_conf['project_id']),
+ "region": "{}".format(odahu_conf['region']),
+ "zone": "{}".format(odahu_conf['zone']),
+ },
+ "type": "gcp"
+ },
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "gcp/gke",
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+
+ "dns": {
+ "domain": "odahu.{}.{}".format(odahu_conf['cluster_name'], odahu_conf['root_domain']),
+ "gcp_project_id": "{}".format(odahu_conf['project_id']),
+ "provider": "gcp",
+ "zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ },
+ "docker_password": "{}".format(odahu_conf['docker_password']),
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_username": "_json_key",
+ "gcp_cidr": "{}".format(odahu_conf['cidr']),
+ "examples_version": "{}".format(odahu_conf['examples_version']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "jupyterhub_enabled": odahu_conf['jupyterhub_enabled'],
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "k8s_version": "{}".format(odahu_conf['k8s_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "node_pools": {
+ "main": {
+ "disk_size_gb": 64,
+ "init_node_count": 3,
+ "max_node_count": 5,
+ "min_node_count": 1
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "deployment"
+ }
+ ]
+ },
+ "packaging": {
+ "disk_size_gb": 64,
+ "disk_type": "pd-ssd",
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "machine_type": "n1-standard-4",
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "packaging"
+ }
+ ]
+ },
+ "training": {
+ "disk_size_gb": 100,
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "machine_type": "n1-highcpu-8",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training"
+ }
+ ]
+ },
+ "training_gpu": {
+ "disk_size_gb": 100,
+ "gpu": [
+ {
+ "count": 2,
+ "type": "nvidia-tesla-p100"
+ }
+ ],
+ "labels": {
+ "mode": "odahu-flow-training-gpu"
+ },
+ "machine_type": "n1-standard-8",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training-gpu"
+ }
+ ]
+ }
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_local_jwks": "{}".format(odahu_conf['oauth_local_jwks']),
+ "oauth_mesh_enabled": odahu_conf['oauth_mesh_enabled'],
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_jwks_url": "{}/protocol/openid-connect/certs".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_port": 443,
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_token_endpoint": "{}/protocol/openid-connect/token".format(odahu_conf['oauth_oidc_issuer_url']),
+ "odahu_automation_version": "{}".format(odahu_conf['automation_version']),
+ "odahu_infra_version": "{}".format(odahu_conf['infra_version']),
+ "odahu_ui_version": "{}".format(odahu_conf['ui_version']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "odahuflow_connections": [
+ {
+ "id": "odahu-flow-examples",
+ "spec": {
+ "description": "Git repository with the Odahu-Flow examples",
+ "keySecret": "{}".format(odahu_conf['keysecret']),
+ "reference": "{}".format(odahu_conf['examples_version']),
+ "type": "git",
+ "uri": "git@github.com:odahu/odahu-examples.git",
+ "webUILink": "https://github.com/odahu/odahu-examples"
+ }
+ }
+ ],
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "opa_policies": {},
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "pods_cidr": "{}".format(odahu_conf['pods_cidr']),
+ "service_accounts": {
+ "airflow": {
+ "client_id": "sa-airflow",
+ "client_secret": "{}".format(odahu_conf['airflow_secret'])
+ },
+ "operator": {
+ "client_id": "sa-operator",
+ "client_secret": "{}".format(odahu_conf['operator_secret'])
+ },
+ "resource_uploader": {
+ "client_id": "sa-resource-uploader",
+ "client_secret": "{}".format(odahu_conf['resource-uploader_secret'])
+ },
+ "test": {
+ "client_id": "sa-tester",
+ "client_secret": "{}".format(odahu_conf['tester_secret'])
+ },
+ "test_data_scientist": {
+ "client_id": "sa-tester-data-scientist",
+ "client_secret": "{}".format(odahu_conf['tester-data-scientist_secret'])
+ }
+ },
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "vault": {
+ "enabled": "true"
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
+ local('cp /tmp/profile.json /')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ GCPActions().remove_bucket(odahu_conf['bucket_name'])
+ GCPActions().remove_static_address(odahu_conf['static_address_name'], odahu_conf['region'])
+ sys.exit(1)
+
+ try:
+ local('tf_runner create -o /tmp/result.json')
+ local("sed -i 's|name|description|g' /tmp/result.json")
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to deploy Odahu cluster.", str(err))
+ GCPActions().remove_bucket(odahu_conf['bucket_name'])
+ GCPActions().remove_static_address(odahu_conf['static_address_name'], odahu_conf['region'])
+ sys.exit(1)
+
+ # generating output information
+ print('[SUMMARY]')
+ logging.info('[SUMMARY]')
+ print('Cluster name: {}'.format(odahu_conf['cluster_name']))
+ with open('/tmp/result.json', 'r') as f:
+ output = json.load(f)
+ odahu_urls = json.dumps(output['odahu_urls']['value'], sort_keys=True, indent=4)
+ print('Odahu urls: {}'.format(odahu_urls))
+ res = dict()
+ res['odahu_urls'] = output['odahu_urls']['value']
+ res['oauth_cookie_secret'] = odahu_conf['oauth_cookie_secret']
+ res['odahuflow_connection_decrypt_token'] = odahu_conf['decrypt_token']
+ res['grafana_pass'] = odahu_conf['grafana_pass']
+ res['grafana_admin'] = odahu_conf['grafana_admin']
+ with open("/root/result.json", 'w') as result:
+ result.write(json.dumps(res))
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_prepare.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_prepare.py
new file mode 100644
index 0000000..1a6c3ab
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_prepare.py
@@ -0,0 +1,142 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+import requests
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['endpoint_name'] = (os.environ['endpoint_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['tag_name'] = '{}-tag'.format(odahu_conf['service_base_name'])
+ odahu_conf['endpoint_tag'] = (os.environ['endpoint_name']).lower().replace('_', '-')
+ odahu_conf['project_tag'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['region'] = os.environ['gcp_region']
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ odahu_conf['keycloak_auth_server_url'] = os.environ['keycloak_auth_server_url']
+ odahu_conf['keycloak_realm_name'] = os.environ['keycloak_realm_name']
+ odahu_conf['keycloak_client_name'] = os.environ['keycloak_client_name']
+ odahu_conf['keycloak_user'] = os.environ['keycloak_user']
+ odahu_conf['keycloak_user_password'] = os.environ['keycloak_user_password']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+
+
+ try:
+ logging.info('[CREATE STATE BUCKETS]')
+ print('[CREATE STATE BUCKETS]')
+
+ odahu_conf['bucket_tags'] = {
+ odahu_conf['tag_name']: odahu_conf['bucket_name'],
+ "endpoint_tag": odahu_conf['endpoint_tag'],
+ os.environ['conf_billing_tag_key']: os.environ['conf_billing_tag_value'],
+ "sbn": odahu_conf['service_base_name'],
+ "project_tag": odahu_conf['project_tag']}
+
+ params = "--bucket_name {} --tags '{}'".format(odahu_conf['bucket_name'], json.dumps(odahu_conf['bucket_tags']))
+
+ try:
+ local("~/scripts/{}.py {}".format('common_create_bucket', params))
+ except:
+ traceback.print_exc()
+ raise Exception
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Unable to create bucket.", str(err))
+ sys.exit(1)
+
+ try:
+ logging.info('[CREATE NAT GATEWAY]')
+ print('[CREATE NAT GATEWAY]')
+ GCPActions().create_static_address(odahu_conf['static_address_name'], odahu_conf['region'])
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Unable to reserve static ip.", str(err))
+ GCPActions().remove_bucket(odahu_conf['bucket_name'])
+ sys.exit(1)
+
+ try:
+ print('[CONFIGURE REDIRECT URI]')
+ logging.info('[CONFIGURE REDIRECT URI]')
+ keycloak_auth_server_url = '{}/realms/master/protocol/openid-connect/token'.format(
+ odahu_conf['keycloak_auth_server_url'])
+ keycloak_auth_data = {
+ "username": odahu_conf['keycloak_user'],
+ "password": odahu_conf['keycloak_user_password'],
+ "grant_type": "password",
+ "client_id": "admin-cli",
+ }
+ keycloak_client_create_url = '{0}/admin/realms/{1}/clients'.format(odahu_conf['keycloak_auth_server_url'],
+ odahu_conf['keycloak_realm_name'])
+ odahu_redirectUris = 'https://odahu.{0}.{1}/*,http://odahu.{0}.{1}/*'.format(odahu_conf['cluster_name'],
+ odahu_conf['root_domain']).split(',')
+
+
+ try:
+ keycloak_token = requests.post(keycloak_auth_server_url, data=keycloak_auth_data, verify=False).json()
+ keycloak_get_Uris = requests.get(keycloak_client_create_url,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"}, verify=False).json()
+ for dict in keycloak_get_Uris:
+ if dict["clientId"] == odahu_conf['keycloak_client_name']:
+ ui_redirectUris = dict["redirectUris"]
+ keycloak_client_id = dict["id"]
+ keycloak_redirectUris = odahu_redirectUris + ui_redirectUris
+ updated_client_data = {
+ "clientId": odahu_conf['keycloak_client_name'],
+ "id": keycloak_client_id,
+ "enabled": "true",
+ "redirectUris": keycloak_redirectUris,
+ "publicClient": "false",
+ "protocol": "openid-connect",
+ }
+ client_url = "{}/{}".format(keycloak_client_create_url, keycloak_client_id)
+ keycloak_update_Uris = requests.put(client_url, json=updated_client_data,
+ headers={"Authorization": "Bearer " + keycloak_token.get("access_token"),
+ "Content-Type": "application/json"}, verify=False)
+ except Exception as err:
+ append_result("Failed to configure keycloak.")
+ sys.exit(1)
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ append_result("Failed to configure keycloak.", str(err))
+ GCPActions().remove_bucket(odahu_conf['bucket_name'])
+ GCPActions().remove_static_address(odahu_conf['static_address_name'], odahu_conf['region'])
+ sys.exit(1)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py
new file mode 100644
index 0000000..994828b
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_resume.py
@@ -0,0 +1,300 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import string
+import random
+import base64
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['gcp_region'])
+ odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['edge_user_name'] = os.environ['edge_user_name']
+ odahu_conf['node_locations'] = GCPMeta().get_available_zones()
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+ odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['grafana_pass']
+ odahu_conf['docker_password'] = base64.b64decode(os.environ['odahu_docker_password'] + "==")
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['keycloak_auth_server_url'].replace('https://', '').replace('/auth', '')
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = os.environ['oauth_cookie_secret']
+ odahu_conf['oauth_local_jwks'] = os.environ['odahu_oauth_local_jwks']
+ odahu_conf['infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['node_version'] = os.environ['odahu_node_version']
+ odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = os.environ['odahuflow_connection_decrypt_token']
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ odahu_conf['automation_version'] = os.environ['odahu_automation_version']
+ odahu_conf['ui_version'] = os.environ['odahu_ui_version']
+ odahu_conf['examples_version'] = os.environ['odahu_examples_version']
+ odahu_conf['jupyterhub_enabled'] = os.environ['odahu_jupyterhub_enabled']
+ odahu_conf['oauth_mesh_enabled'] = os.environ['odahu_oauth_mesh_enabled']
+ odahu_conf['keysecret'] = os.environ['odahu_keysecret']
+ odahu_conf['airflow_secret'] = os.environ['odahu_airflow_secret']
+ odahu_conf['operator_secret'] = os.environ['odahu_operator_secret']
+ odahu_conf['resource-uploader_secret'] = os.environ['odahu_resource_uploader_secret']
+ odahu_conf['tester_secret'] = os.environ['odahu_tester_secret']
+ odahu_conf['tester-data-scientist_secret'] = os.environ['odahu_tester_data_scientist_secret']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {
+ "allowed_ips": odahu_conf['allowed_cidr'],
+ "authorization_enabled": "true",
+ "authz_dry_run": "false",
+ "cloud": {
+ "gcp": {
+ "node_locations": odahu_conf['node_locations'],
+ "project_id": "{}".format(odahu_conf['project_id']),
+ "region": "{}".format(odahu_conf['region']),
+ "zone": "{}".format(odahu_conf['zone']),
+ },
+ "type": "gcp"
+ },
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "gcp/gke",
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+
+ "dns": {
+ "domain": "odahu.{}.{}".format(odahu_conf['cluster_name'], odahu_conf['root_domain']),
+ "gcp_project_id": "{}".format(odahu_conf['project_id']),
+ "provider": "gcp",
+ "zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ },
+ "docker_password": "{}".format(odahu_conf['docker_password']),
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_username": "_json_key",
+ "gcp_cidr": "{}".format(odahu_conf['cidr']),
+ "examples_version": "{}".format(odahu_conf['examples_version']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "jupyterhub_enabled": odahu_conf['jupyterhub_enabled'],
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "k8s_version": "{}".format(odahu_conf['k8s_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "node_pools": {
+ "main": {
+ "disk_size_gb": 64,
+ "init_node_count": 8,
+ "max_node_count": 5,
+ "min_node_count": 1
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "deployment"
+ }
+ ]
+ },
+ "packaging": {
+ "disk_size_gb": 64,
+ "disk_type": "pd-ssd",
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "machine_type": "n1-standard-4",
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "packaging"
+ }
+ ]
+ },
+ "training": {
+ "disk_size_gb": 100,
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "machine_type": "n1-highcpu-8",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training"
+ }
+ ]
+ },
+ "training_gpu": {
+ "disk_size_gb": 100,
+ "gpu": [
+ {
+ "count": 2,
+ "type": "nvidia-tesla-p100"
+ }
+ ],
+ "labels": {
+ "mode": "odahu-flow-training-gpu"
+ },
+ "machine_type": "n1-standard-8",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training-gpu"
+ }
+ ]
+ }
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_local_jwks": "{}".format(odahu_conf['oauth_local_jwks']),
+ "oauth_mesh_enabled": odahu_conf['oauth_mesh_enabled'],
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_jwks_url": "{}/protocol/openid-connect/certs".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_port": 443,
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_token_endpoint": "{}/protocol/openid-connect/token".format(
+ odahu_conf['oauth_oidc_issuer_url']),
+ "odahu_automation_version": "{}".format(odahu_conf['automation_version']),
+ "odahu_infra_version": "{}".format(odahu_conf['infra_version']),
+ "odahu_ui_version": "{}".format(odahu_conf['ui_version']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "odahuflow_connections": [
+ {
+ "id": "odahu-flow-examples",
+ "spec": {
+ "description": "Git repository with the Odahu-Flow examples",
+ "keySecret": "{}".format(odahu_conf['keysecret']),
+ "reference": "{}".format(odahu_conf['examples_version']),
+ "type": "git",
+ "uri": "git@github.com:odahu/odahu-examples.git",
+ "webUILink": "https://github.com/odahu/odahu-examples"
+ }
+ }
+ ],
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "opa_policies": {},
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "pods_cidr": "{}".format(odahu_conf['pods_cidr']),
+ "service_accounts": {
+ "airflow": {
+ "client_id": "sa-airflow",
+ "client_secret": "{}".format(odahu_conf['airflow_secret'])
+ },
+ "operator": {
+ "client_id": "sa-operator",
+ "client_secret": "{}".format(odahu_conf['operator_secret'])
+ },
+ "resource_uploader": {
+ "client_id": "sa-resource-uploader",
+ "client_secret": "{}".format(odahu_conf['resource-uploader_secret'])
+ },
+ "test": {
+ "client_id": "sa-tester",
+ "client_secret": "{}".format(odahu_conf['tester_secret'])
+ },
+ "test_data_scientist": {
+ "client_id": "sa-tester-data-scientist",
+ "client_secret": "{}".format(odahu_conf['tester-data-scientist_secret'])
+ }
+ },
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "vault": {
+ "enabled": "true"
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
+ local('cp /tmp/profile.json /')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
+
+ try:
+ local('tf_runner resume -v')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to deploy Odahu cluster.", str(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py
new file mode 100644
index 0000000..54aff9e
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_suspend.py
@@ -0,0 +1,298 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import base64
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['gcp_region'])
+ odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['edge_user_name'] = os.environ['edge_user_name']
+ odahu_conf['node_locations'] = GCPMeta().get_available_zones()
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+ odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['grafana_pass']
+ odahu_conf['docker_password'] = base64.b64decode(os.environ['odahu_docker_password'] + "==")
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['keycloak_auth_server_url'].replace('https://', '').replace('/auth', '')
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = os.environ['oauth_cookie_secret']
+ odahu_conf['oauth_local_jwks'] = os.environ['odahu_oauth_local_jwks']
+ odahu_conf['infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['node_version'] = os.environ['odahu_node_version']
+ odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = os.environ['odahuflow_connection_decrypt_token']
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ odahu_conf['automation_version'] = os.environ['odahu_automation_version']
+ odahu_conf['ui_version'] = os.environ['odahu_ui_version']
+ odahu_conf['examples_version'] = os.environ['odahu_examples_version']
+ odahu_conf['jupyterhub_enabled'] = os.environ['odahu_jupyterhub_enabled']
+ odahu_conf['oauth_mesh_enabled'] = os.environ['odahu_oauth_mesh_enabled']
+ odahu_conf['keysecret'] = os.environ['odahu_keysecret']
+ odahu_conf['airflow_secret'] = os.environ['odahu_airflow_secret']
+ odahu_conf['operator_secret'] = os.environ['odahu_operator_secret']
+ odahu_conf['resource-uploader_secret'] = os.environ['odahu_resource_uploader_secret']
+ odahu_conf['tester_secret'] = os.environ['odahu_tester_secret']
+ odahu_conf['tester-data-scientist_secret'] = os.environ['odahu_tester_data_scientist_secret']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {
+ "allowed_ips": odahu_conf['allowed_cidr'],
+ "authorization_enabled": "true",
+ "authz_dry_run": "false",
+ "cloud": {
+ "gcp": {
+ "node_locations": odahu_conf['node_locations'],
+ "project_id": "{}".format(odahu_conf['project_id']),
+ "region": "{}".format(odahu_conf['region']),
+ "zone": "{}".format(odahu_conf['zone']),
+ },
+ "type": "gcp"
+ },
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "gcp/gke",
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+
+ "dns": {
+ "domain": "odahu.{}.{}".format(odahu_conf['cluster_name'], odahu_conf['root_domain']),
+ "gcp_project_id": "{}".format(odahu_conf['project_id']),
+ "provider": "gcp",
+ "zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ },
+ "docker_password": "{}".format(odahu_conf['docker_password']),
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_username": "_json_key",
+ "gcp_cidr": "{}".format(odahu_conf['cidr']),
+ "examples_version": "{}".format(odahu_conf['examples_version']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "jupyterhub_enabled": odahu_conf['jupyterhub_enabled'],
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "k8s_version": "{}".format(odahu_conf['k8s_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "node_pools": {
+ "main": {
+ "disk_size_gb": 64,
+ "init_node_count": 3,
+ "max_node_count": 5,
+ "min_node_count": 1
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "deployment"
+ }
+ ]
+ },
+ "packaging": {
+ "disk_size_gb": 64,
+ "disk_type": "pd-ssd",
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "machine_type": "n1-standard-4",
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "packaging"
+ }
+ ]
+ },
+ "training": {
+ "disk_size_gb": 100,
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "machine_type": "n1-highcpu-8",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training"
+ }
+ ]
+ },
+ "training_gpu": {
+ "disk_size_gb": 100,
+ "gpu": [
+ {
+ "count": 2,
+ "type": "nvidia-tesla-p100"
+ }
+ ],
+ "labels": {
+ "mode": "odahu-flow-training-gpu"
+ },
+ "machine_type": "n1-standard-8",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training-gpu"
+ }
+ ]
+ }
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_local_jwks": "{}".format(odahu_conf['oauth_local_jwks']),
+ "oauth_mesh_enabled": odahu_conf['oauth_mesh_enabled'],
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_jwks_url": "{}/protocol/openid-connect/certs".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_port": 443,
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_token_endpoint": "{}/protocol/openid-connect/token".format(
+ odahu_conf['oauth_oidc_issuer_url']),
+ "odahu_automation_version": "{}".format(odahu_conf['automation_version']),
+ "odahu_infra_version": "{}".format(odahu_conf['infra_version']),
+ "odahu_ui_version": "{}".format(odahu_conf['ui_version']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "odahuflow_connections": [
+ {
+ "id": "odahu-flow-examples",
+ "spec": {
+ "description": "Git repository with the Odahu-Flow examples",
+ "keySecret": "{}".format(odahu_conf['keysecret']),
+ "reference": "{}".format(odahu_conf['examples_version']),
+ "type": "git",
+ "uri": "git@github.com:odahu/odahu-examples.git",
+ "webUILink": "https://github.com/odahu/odahu-examples"
+ }
+ }
+ ],
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "opa_policies": {},
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "pods_cidr": "{}".format(odahu_conf['pods_cidr']),
+ "service_accounts": {
+ "airflow": {
+ "client_id": "sa-airflow",
+ "client_secret": "{}".format(odahu_conf['airflow_secret'])
+ },
+ "operator": {
+ "client_id": "sa-operator",
+ "client_secret": "{}".format(odahu_conf['operator_secret'])
+ },
+ "resource_uploader": {
+ "client_id": "sa-resource-uploader",
+ "client_secret": "{}".format(odahu_conf['resource-uploader_secret'])
+ },
+ "test": {
+ "client_id": "sa-tester",
+ "client_secret": "{}".format(odahu_conf['tester_secret'])
+ },
+ "test_data_scientist": {
+ "client_id": "sa-tester-data-scientist",
+ "client_secret": "{}".format(odahu_conf['tester-data-scientist_secret'])
+ }
+ },
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "vault": {
+ "enabled": "true"
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
+ local('cp /tmp/profile.json /')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
+
+ try:
+ local('tf_runner suspend -v')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to suspend Odahu cluster.", str(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py b/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
new file mode 100644
index 0000000..c964ce3
--- /dev/null
+++ b/infrastructure-provisioning/src/general/scripts/gcp/odahu_terminate.py
@@ -0,0 +1,318 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import base64
+
+
+if __name__ == "__main__":
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['project_name'],
+ os.environ['request_id'])
+ local_log_filepath = "/logs/project/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ print('Generating infrastructure names and tags')
+ odahu_conf = dict()
+ odahu_conf['allowed_cidr'] = os.environ['odahu_allowed_cidr'].split(',')
+ odahu_conf['project_id'] = (os.environ['gcp_project_id'])
+ odahu_conf['region'] = (os.environ['gcp_region'])
+ odahu_conf['zone'] = (os.environ['gcp_zone'])
+ odahu_conf['edge_user_name'] = os.environ['edge_user_name']
+ odahu_conf['node_locations'] = GCPMeta().get_available_zones()
+ odahu_conf['dns_zone_name'] = os.environ['odahu_dns_zone_name']
+ odahu_conf['docker_repo'] = os.environ['odahu_docker_repo']
+ odahu_conf['cidr'] = os.environ['odahu_cidr']
+ odahu_conf['service_base_name'] = (os.environ['conf_service_base_name']).lower().replace('_', '-')
+ odahu_conf['project_name'] = (os.environ['project_name']).lower().replace('_', '-')
+ odahu_conf['cluster_name'] = "{}-{}".format((os.environ['conf_service_base_name']).lower().replace('_', '-'),
+ (os.environ['odahu_cluster_name']).lower().replace('_', '-'))
+ odahu_conf['bucket_name'] = "{}-tfstate".format(odahu_conf['cluster_name'])
+ odahu_conf['static_address_name'] = "{}-nat-gw".format(odahu_conf['cluster_name'])
+ try:
+ if os.environ['gcp_vpc_name'] == '':
+ raise KeyError
+ else:
+ odahu_conf['vpc_name'] = os.environ['gcp_vpc_name']
+ except KeyError:
+ odahu_conf['vpc_name'] = odahu_conf['service_base_name'] + '-ssn-vpc'
+ odahu_conf['vpc_cidr'] = os.environ['conf_vpc_cidr']
+ odahu_conf['private_subnet_name'] = '{0}-{1}-subnet'.format(odahu_conf['service_base_name'],
+ odahu_conf['project_name'])
+ odahu_conf['grafana_admin'] = os.environ['grafana_admin']
+ odahu_conf['grafana_pass'] = os.environ['grafana_pass']
+ odahu_conf['docker_password'] = base64.b64decode(os.environ['odahu_docker_password'] + "==")
+ odahu_conf['initial_node_count'] = os.environ['odahu_initial_node_count']
+ odahu_conf['istio_helm_repo'] = os.environ['odahu_istio_helm_repo']
+ odahu_conf['helm_repo'] = os.environ['odahu_helm_repo']
+ odahu_conf['k8s_version'] = os.environ['odahu_k8s_version']
+ odahu_conf['oauth_oidc_issuer_url'] = "{}/realms/{}".format(os.environ['keycloak_auth_server_url'],
+ os.environ['keycloak_realm_name'])
+ odahu_conf['oauth_oidc_host'] = os.environ['keycloak_auth_server_url'].replace('https://', '').replace('/auth', '')
+ odahu_conf['oauth_client_id'] = os.environ['keycloak_client_name']
+ odahu_conf['oauth_client_secret'] = os.environ['keycloak_client_secret']
+ odahu_conf['oauth_cookie_secret'] = os.environ['oauth_cookie_secret']
+ odahu_conf['oauth_local_jwks'] = os.environ['odahu_oauth_local_jwks']
+ odahu_conf['infra_version'] = os.environ['odahu_infra_version']
+ odahu_conf['odahuflow_version'] = os.environ['odahu_odahuflow_version']
+ odahu_conf['mlflow_toolchain_version'] = os.environ['odahu_mlflow_toolchain_version']
+ odahu_conf['jupyterlab_version'] = os.environ['odahu_jupyterlab_version']
+ odahu_conf['packager_version'] = os.environ['odahu_packager_version']
+ odahu_conf['node_version'] = os.environ['odahu_node_version']
+ odahu_conf['pods_cidr'] = os.environ['odahu_pods_cidr']
+ odahu_conf['root_domain'] = os.environ['odahu_root_domain']
+ odahu_conf['service_cidr'] = os.environ['odahu_service_cidr']
+ odahu_conf['tls_crt'] = base64.b64decode(os.environ['odahu_tls_crt'] + "==")
+ odahu_conf['tls_key'] = base64.b64decode(os.environ['odahu_tls_key'] + "==")
+ odahu_conf['ssh_key'] = os.environ['ssh_key']
+ odahu_conf['dns_project_id'] = os.environ['odahu_dns_project_id']
+ odahu_conf['decrypt_token'] = os.environ['odahuflow_connection_decrypt_token']
+ odahu_conf['infra_vpc_peering'] = os.environ['odahu_infra_vpc_peering']
+ odahu_conf['automation_version'] = os.environ['odahu_automation_version']
+ odahu_conf['ui_version'] = os.environ['odahu_ui_version']
+ odahu_conf['examples_version'] = os.environ['odahu_examples_version']
+ odahu_conf['jupyterhub_enabled'] = os.environ['odahu_jupyterhub_enabled']
+ odahu_conf['oauth_mesh_enabled'] = os.environ['odahu_oauth_mesh_enabled']
+ odahu_conf['keysecret'] = os.environ['odahu_keysecret']
+ odahu_conf['airflow_secret'] = os.environ['odahu_airflow_secret']
+ odahu_conf['operator_secret'] = os.environ['odahu_operator_secret']
+ odahu_conf['resource-uploader_secret'] = os.environ['odahu_resource_uploader_secret']
+ odahu_conf['tester_secret'] = os.environ['odahu_tester_secret']
+ odahu_conf['tester-data-scientist_secret'] = os.environ['odahu_tester_data_scientist_secret']
+
+ print('Preparing parameters file')
+ try:
+ local("cp /root/templates/profile.json /tmp/")
+ with open("/tmp/profile.json", 'w') as profile:
+ prof = {
+ "allowed_ips": odahu_conf['allowed_cidr'],
+ "authorization_enabled": "true",
+ "authz_dry_run": "false",
+ "cloud": {
+ "gcp": {
+ "node_locations": odahu_conf['node_locations'],
+ "project_id": "{}".format(odahu_conf['project_id']),
+ "region": "{}".format(odahu_conf['region']),
+ "zone": "{}".format(odahu_conf['zone']),
+ },
+ "type": "gcp"
+ },
+ "cluster_name": "{}".format(odahu_conf['cluster_name']),
+ "cluster_type": "gcp/gke",
+ "data_bucket": "{}-data-bucket".format(odahu_conf['cluster_name']),
+
+ "dns": {
+ "domain": "odahu.{}.{}".format(odahu_conf['cluster_name'], odahu_conf['root_domain']),
+ "gcp_project_id": "{}".format(odahu_conf['project_id']),
+ "provider": "gcp",
+ "zone_name": "{}".format(odahu_conf['dns_zone_name']),
+ },
+ "docker_password": "{}".format(odahu_conf['docker_password']),
+ "docker_repo": "{}".format(odahu_conf['docker_repo']),
+ "docker_username": "_json_key",
+ "gcp_cidr": "{}".format(odahu_conf['cidr']),
+ "examples_version": "{}".format(odahu_conf['examples_version']),
+ "grafana_pass": "{}".format(odahu_conf['grafana_pass']),
+ "helm_repo": "{}".format(odahu_conf['helm_repo']),
+ "jupyterhub_enabled": odahu_conf['jupyterhub_enabled'],
+ "jupyterlab_version": "{}".format(odahu_conf['jupyterlab_version']),
+ "k8s_version": "{}".format(odahu_conf['k8s_version']),
+ "mlflow_toolchain_version": "{}".format(odahu_conf['mlflow_toolchain_version']),
+ "node_pools": {
+ "main": {
+ "disk_size_gb": 64,
+ "init_node_count": 3,
+ "max_node_count": 5,
+ "min_node_count": 1
+ },
+ "model_deployment": {
+ "labels": {
+ "mode": "odahu-flow-deployment"
+ },
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "deployment"
+ }
+ ]
+ },
+ "packaging": {
+ "disk_size_gb": 64,
+ "disk_type": "pd-ssd",
+ "labels": {
+ "mode": "odahu-flow-packaging"
+ },
+ "machine_type": "n1-standard-4",
+ "max_node_count": 3,
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "packaging"
+ }
+ ]
+ },
+ "training": {
+ "disk_size_gb": 100,
+ "labels": {
+ "mode": "odahu-flow-training"
+ },
+ "machine_type": "n1-highcpu-8",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training"
+ }
+ ]
+ },
+ "training_gpu": {
+ "disk_size_gb": 100,
+ "gpu": [
+ {
+ "count": 2,
+ "type": "nvidia-tesla-p100"
+ }
+ ],
+ "labels": {
+ "mode": "odahu-flow-training-gpu"
+ },
+ "machine_type": "n1-standard-8",
+ "taints": [
+ {
+ "effect": "NO_SCHEDULE",
+ "key": "dedicated",
+ "value": "training-gpu"
+ }
+ ]
+ }
+ },
+ "oauth_client_id": "{}".format(odahu_conf['oauth_client_id']),
+ "oauth_client_secret": "{}".format(odahu_conf['oauth_client_secret']),
+ "oauth_cookie_secret": "{}".format(odahu_conf['oauth_cookie_secret']),
+ "oauth_local_jwks": "{}".format(odahu_conf['oauth_local_jwks']),
+ "oauth_mesh_enabled": odahu_conf['oauth_mesh_enabled'],
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_host": "{}".format(odahu_conf['oauth_oidc_host']),
+ "oauth_oidc_issuer_url": "{}".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_jwks_url": "{}/protocol/openid-connect/certs".format(odahu_conf['oauth_oidc_issuer_url']),
+ "oauth_oidc_port": 443,
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "oauth_oidc_token_endpoint": "{}/protocol/openid-connect/token".format(
+ odahu_conf['oauth_oidc_issuer_url']),
+ "odahu_automation_version": "{}".format(odahu_conf['automation_version']),
+ "odahu_infra_version": "{}".format(odahu_conf['infra_version']),
+ "odahu_ui_version": "{}".format(odahu_conf['ui_version']),
+ "odahuflow_connection_decrypt_token": "{}".format(odahu_conf['decrypt_token']),
+ "odahuflow_connections": [
+ {
+ "id": "odahu-flow-examples",
+ "spec": {
+ "description": "Git repository with the Odahu-Flow examples",
+ "keySecret": "{}".format(odahu_conf['keysecret']),
+ "reference": "{}".format(odahu_conf['examples_version']),
+ "type": "git",
+ "uri": "git@github.com:odahu/odahu-examples.git",
+ "webUILink": "https://github.com/odahu/odahu-examples"
+ }
+ }
+ ],
+ "odahuflow_version": "{}".format(odahu_conf['odahuflow_version']),
+ "opa_policies": {},
+ "packager_version": "{}".format(odahu_conf['packager_version']),
+ "pods_cidr": "{}".format(odahu_conf['pods_cidr']),
+ "service_accounts": {
+ "airflow": {
+ "client_id": "sa-airflow",
+ "client_secret": "{}".format(odahu_conf['airflow_secret'])
+ },
+ "operator": {
+ "client_id": "sa-operator",
+ "client_secret": "{}".format(odahu_conf['operator_secret'])
+ },
+ "resource_uploader": {
+ "client_id": "sa-resource-uploader",
+ "client_secret": "{}".format(odahu_conf['resource-uploader_secret'])
+ },
+ "test": {
+ "client_id": "sa-tester",
+ "client_secret": "{}".format(odahu_conf['tester_secret'])
+ },
+ "test_data_scientist": {
+ "client_id": "sa-tester-data-scientist",
+ "client_secret": "{}".format(odahu_conf['tester-data-scientist_secret'])
+ }
+ },
+ "service_cidr": "{}".format(odahu_conf['service_cidr']),
+ "ssh_key": "{}".format(odahu_conf['ssh_key'].replace('\n', '')),
+ "subnet_name": "{}".format(odahu_conf['private_subnet_name']),
+ "tfstate_bucket": "{}-tfstate".format(odahu_conf['cluster_name']),
+ "tls_crt": "{}".format(odahu_conf['tls_crt']),
+ "tls_key": "{}".format(odahu_conf['tls_key']),
+ "vpc_name": "{}".format(odahu_conf['vpc_name']),
+ "vault": {
+ "enabled": "true"
+ }
+ }
+ profile.write(json.dumps(prof))
+ local('cat /tmp/profile.json')
+ local('cp /tmp/profile.json /')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to configure parameter file.", str(err))
+ sys.exit(1)
+
+ print('Removing Odahu cluster')
+ try:
+ local('tf_runner destroy')
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to terminate Odahu cluster.", str(err))
+ sys.exit(1)
+
+ try:
+ buckets = GCPMeta().get_list_buckets(odahu_conf['cluster_name'])
+ if 'items' in buckets:
+ for i in buckets['items']:
+ GCPActions().remove_bucket(i['name'])
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ sys.exit(1)
+
+ try:
+ static_addresses = GCPMeta().get_list_static_addresses(odahu_conf['region'], odahu_conf['cluster_name'])
+ if 'items' in static_addresses:
+ for i in static_addresses['items']:
+ GCPActions().remove_static_address(i['name'], odahu_conf['region'])
+ except Exception as err:
+ print('Error: {0}'.format(err))
+ sys.exit(1)
diff --git a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
index 4a10115..a47e53b 100644
--- a/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
+++ b/infrastructure-provisioning/src/general/scripts/gcp/ssn_configure.py
@@ -202,6 +202,7 @@
additional_config = [{"name": "base", "tag": "latest"},
{"name": "project", "tag": "latest"},
{"name": "edge", "tag": "latest"},
+ {"name": "odahu", "tag": "latest"},
{"name": "jupyter", "tag": "latest"},
{"name": "jupyterlab", "tag": "latest"},
{"name": "rstudio", "tag": "latest"},
@@ -213,11 +214,10 @@
{"name": "dataengine", "tag": "latest"},
{"name": "dataengine-service", "tag": "latest"}]
params = "--hostname {} --keyfile {} --additional_config '{}' --os_family {} --os_user {} --dlab_path {} " \
- "--cloud_provider {} --region {}". \
+ "--cloud_provider {} --region {} --gcr_creds {} --odahu_image {}". \
format(ssn_conf['instance_hostname'], ssn_conf['ssh_key_path'], json.dumps(additional_config),
os.environ['conf_os_family'], ssn_conf['dlab_ssh_user'], os.environ['ssn_dlab_path'],
- os.environ['conf_cloud_provider'], ssn_conf['region'])
-
+ os.environ['conf_cloud_provider'], ssn_conf['region'], os.environ['ssn_gcr_creds'], os.environ['odahu_deploy_image'])
try:
local("~/scripts/{}.py {}".format('configure_docker', params))
except:
diff --git a/infrastructure-provisioning/src/odahu/fabfile.py b/infrastructure-provisioning/src/odahu/fabfile.py
new file mode 100644
index 0000000..85b09fb
--- /dev/null
+++ b/infrastructure-provisioning/src/odahu/fabfile.py
@@ -0,0 +1,98 @@
+#!/usr/bin/python
+
+# *****************************************************************************
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# ******************************************************************************
+
+import logging
+import json
+import sys
+from dlab.fab import *
+from dlab.meta_lib import *
+from dlab.actions_lib import *
+import os
+import uuid
+
+
+# Main function for provisioning Odahuflow cluster
+def run():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+
+ try:
+ local("~/scripts/{}.py".format('odahu_prepare'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed preparing Notebook node.", str(err))
+ sys.exit(1)
+
+ try:
+ local("~/scripts/{}.py".format('odahu_deploy'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to deploy Odahuflow cluster.", str(err))
+ sys.exit(1)
+
+# Main function for Odahuflow cluster termination
+def terminate():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+ try:
+ local("~/scripts/{}.py".format('odahu_terminate'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to terminate Odahuflow cluster.", str(err))
+ sys.exit(1)
+
+
+# Main function for suspending Odahuflow cluster
+def stop():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+ try:
+ local("~/scripts/{}.py".format('odahu_suspend'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to suspend Odahuflow cluster.", str(err))
+ sys.exit(1)
+
+
+# Main function for resuming Odahuflow cluster
+def start():
+ local_log_filename = "{}_{}_{}.log".format(os.environ['conf_resource'], os.environ['edge_user_name'], os.environ['request_id'])
+ local_log_filepath = "/logs/" + os.environ['conf_resource'] + "/" + local_log_filename
+ logging.basicConfig(format='%(levelname)-8s [%(asctime)s] %(message)s',
+ level=logging.DEBUG,
+ filename=local_log_filepath)
+ try:
+ local("~/scripts/{}.py".format('odahu_resume'))
+ except Exception as err:
+ traceback.print_exc()
+ append_result("Failed to resume Odahuflow cluster.", str(err))
+ sys.exit(1)
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/odahu/templates/profile.json b/infrastructure-provisioning/src/odahu/templates/profile.json
new file mode 100644
index 0000000..a4b2232
--- /dev/null
+++ b/infrastructure-provisioning/src/odahu/templates/profile.json
@@ -0,0 +1,73 @@
+{
+ "alert_slack_url": "",
+ "allowed_ips": <ALLOWED_IP_CIDR>,
+ "aws_cidr": "",
+ "aws_profile": "",
+ "aws_route_table_id": "",
+ "aws_sg": "",
+ "aws_vpc_id": "",
+ "bastion_tag": "<BASTION_TAG>",
+ "cloud_type": "gcp",
+ "cluster_context": "gke_<PROJECT_ID>_<REGION>_<CLUSTER_NAME>",
+ "cluster_name": "<CLUSTER_NAME>",
+ "cluster_type": "gcp/gke",
+ "config_context_auth_info": "gke_<PROJECT_ID>_<REGION>_<CLUSTER_NAME>",
+ "config_context_cluster": "gke_<PROJECT_ID>_<REGION>_<CLUSTER_NAME>",
+ "dns_zone_name": "<DNS_ZONE_NAME>",
+ "docker_password": "",
+ "docker_repo": "<DOCKER_REPO>",
+ "docker_user": "",
+ "enclave_jwt_secret": "",
+ "gcp_cidr": "<ODAHU_CIDR>",
+ "git_examples_description": "",
+ "git_examples_key": "",
+ "git_examples_reference": "",
+ "git_examples_uri": "",
+ "git_examples_web_ui_link": "",
+ "github_org_name": "",
+ "gke_node_tag": "<CLUSTER_NAME>-gke-node",
+ "grafana_admin": "<GRAFANA_ADMIN>",
+ "grafana_pass": "<GRAFANA_PASS>",
+ "infra_cidr": "",
+ "initial_node_count": "<INITIAL_NODE_COUNT>",
+ "istio_helm_repo": "<ISTIO_HELM_REPO>",
+ "k8s_version": "<K8S_VERSION>",
+ "location": "<REGION>",
+ "oauth_oidc_audience": "legion",
+ "oauth_oidc_issuer_url": "<OAUTH_OIDC_ISSUER_URL>",
+ "data_bucket": "<CLUSTER_NAME>-data-bucket",
+ "helm_repo": "<HELM_REPO>",
+ "odahu_infra_version": "<ODAHU_INFRA_VERSION>",
+ "odahuflow_version": "<ODAHUFLOW_VERSION>",
+ "mlflow_toolchain_version": "<MLFLOW_TOOLCHAIN_VERSION>",
+ "jupyterlab_version": "<JUPYTERLAB_VERSION>",
+ "packager_version": "<PACKAGER_VERSION>",
+ "vpc_name": "<VPC_NAME>",
+ "network_name": "<VPC_NAME>",
+ "subnet_name": "<SUBNET_NAME>",
+ "node_locations": <NODE_LOCATIONS>,
+ "node_version": "<NODE_VERSION>",
+ "oauth_client_id": "<OAUTH_CLIENT_ID>",
+ "oauth_client_secret": "<OAUTH_CLIENT_SECRET>",
+ "oauth_cookie_secret": "<OAUTH_COOCKIE_SECRET>",
+ "oauth_oidc_scope": "openid profile email offline_access groups",
+ "pods_cidr": "<PODS_CIDR>",
+ "project_id": "<PROJECT_ID>",
+ "region": "<REGION>",
+ "root_domain": "<ROOT_DOMAIN>",
+ "service_cidr": "<SERVICE_CIDR>",
+ "ssh_key": "<SSH_KEY>",
+ "test_oauth_auth_url": "",
+ "test_oauth_client_id": "",
+ "test_oauth_client_secret": "",
+ "test_oauth_scope": "",
+ "test_user_email": "",
+ "test_user_password": "",
+ "tfstate_bucket": "<CLUSTER_NAME>-tfstate",
+ "tls_crt": "<TLS_CRT>",
+ "tls_key": "<TLS_KEY>",
+ "zone": "<ZONE>",
+ "dns_project_id": "<DNS_PROJECT_ID>",
+ "odahuflow_connection_decrypt_token": "<DECRYPT_TOKEN>",
+ "infra_vpc_peering": "<INFRA_VPC_PEERING>"
+}
\ No newline at end of file
diff --git a/infrastructure-provisioning/src/ssn/files/aws/mongo_roles.json b/infrastructure-provisioning/src/ssn/files/aws/mongo_roles.json
index 54d2cd6..00d4821 100644
--- a/infrastructure-provisioning/src/ssn/files/aws/mongo_roles.json
+++ b/infrastructure-provisioning/src/ssn/files/aws/mongo_roles.json
@@ -279,7 +279,8 @@
"/api/settings",
"/user/settings",
"/api/project",
- "/api/endpoint"
+ "/api/endpoint",
+ "/api/odahu"
],
"groups": [
"$anyuser"
diff --git a/infrastructure-provisioning/src/ssn/files/azure/mongo_roles.json b/infrastructure-provisioning/src/ssn/files/azure/mongo_roles.json
index 305c46b..a001da2 100644
--- a/infrastructure-provisioning/src/ssn/files/azure/mongo_roles.json
+++ b/infrastructure-provisioning/src/ssn/files/azure/mongo_roles.json
@@ -229,7 +229,8 @@
"/api/settings",
"/user/settings",
"/api/project",
- "/api/endpoint"
+ "/api/endpoint",
+ "/api/odahu"
],
"groups": [
"$anyuser"
diff --git a/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json b/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json
index 43d12e3..67548bf 100644
--- a/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json
+++ b/infrastructure-provisioning/src/ssn/files/gcp/mongo_roles.json
@@ -259,7 +259,8 @@
"/api/settings",
"/user/settings",
"/api/project",
- "/api/endpoint"
+ "/api/endpoint",
+ "/api/odahu"
],
"groups": [
"$anyuser"
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
index 727f97e..86e9e65 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_docker.py
@@ -28,6 +28,7 @@
from dlab.ssn_lib import *
import os
import time
+import base64
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', type=str, default='')
@@ -39,6 +40,8 @@
parser.add_argument('--cloud_provider', type=str, default='')
parser.add_argument('--resource', type=str, default='')
parser.add_argument('--region', type=str, default='')
+parser.add_argument('--gcr_creds', type=str, default='')
+parser.add_argument('--odahu_image', type=str, default='')
args = parser.parse_args()
@@ -78,6 +81,31 @@
sudo('sed -i "/pip install/s/jupyter/ipython==5.0.0 jupyter==1.0.0/g" Dockerfile')
sudo('sed -i "22i COPY general/files/os/debian/sources.list /etc/apt/sources.list" Dockerfile')
+def login_in_gcr(os_user, gcr_creds, odahu_image, dlab_path, os_family):
+ if os.environ['conf_cloud_provider'] != 'gcp':
+ try:
+ sudo('echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt '
+ 'cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list')
+ sudo('apt-get -y install apt-transport-https ca-certificates gnupg')
+ sudo('curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -')
+ sudo('apt-get update')
+ sudo('apt-get -y install google-cloud-sdk')
+ except Exception as err:
+ traceback.print_exc()
+ print('Failed to install gcloud: ', str(err))
+ sys.exit(1)
+ try:
+ with open('/tmp/config', 'w') as f:
+ f.write(base64.b64decode(gcr_creds))
+ local('scp -i {} /tmp/config {}:/tmp/config'.format(args.keyfile, env.host_string, os_user))
+ sudo('mkdir /home/{}/.docker'.format(os_user))
+ sudo('cp /tmp/config /home/{}/.docker/config.json'.format(os_user))
+ sudo('sed -i "s|ODAHU_IMAGE|{}|" {}sources/infrastructure-provisioning/src/general/files/{}/odahu_Dockerfile'
+ .format(odahu_image, dlab_path, os_family))
+ except Exception as err:
+ traceback.print_exc()
+ print('Failed to prepare odahu image: ', str(err))
+ sys.exit(1)
def build_docker_images(image_list, region, dlab_path):
try:
@@ -176,6 +204,9 @@
if not ensure_docker_daemon(args.dlab_path, args.os_user, args.region):
sys.exit(1)
+ print("Login in Google Container Registry")
+ login_in_gcr(args.os_user, args.gcr_creds, args.odahu_image, args.dlab_path, args.os_family)
+
print("Building dlab images")
count = 0
while not build_docker_images(deeper_config, args.region, args.dlab_path) and count < 5:
diff --git a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
index 42a6a09..1316542 100644
--- a/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
+++ b/infrastructure-provisioning/src/ssn/scripts/configure_ssn_node.py
@@ -199,6 +199,7 @@
print('Failed to configure docker_build script: ', str(err))
sys.exit(1)
+
##############
# Run script #
##############
@@ -262,4 +263,4 @@
ensure_ciphers()
print("Configuring docker_build script")
- docker_build_script()
\ No newline at end of file
+ docker_build_script()
diff --git a/infrastructure-provisioning/src/ssn/templates/daemon.json b/infrastructure-provisioning/src/ssn/templates/daemon.json
new file mode 100644
index 0000000..e2257ae
--- /dev/null
+++ b/infrastructure-provisioning/src/ssn/templates/daemon.json
@@ -0,0 +1,5 @@
+{
+ "insecure-registries":["<NEXUS_URL>"],
+ "registry-mirrors": ["http://<NEXUS_URL>"],
+ "disable-legacy-registry": true
+}
\ No newline at end of file
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/ResourceURL.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/ResourceURL.java
index fac4891..44c07e3 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/dto/ResourceURL.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/ResourceURL.java
@@ -19,6 +19,7 @@
package com.epam.dlab.dto;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
@@ -30,6 +31,7 @@
@Data
@AllArgsConstructor
@NoArgsConstructor
+@JsonIgnoreProperties(ignoreUnknown = true)
public class ResourceURL {
@JsonProperty("description")
private String description;
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/base/odahu/OdahuResult.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/base/odahu/OdahuResult.java
new file mode 100644
index 0000000..88349ed
--- /dev/null
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/base/odahu/OdahuResult.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.dto.base.odahu;
+
+import com.epam.dlab.dto.ResourceURL;
+import com.epam.dlab.dto.StatusBaseDTO;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class OdahuResult extends StatusBaseDTO<OdahuResult> {
+ private String name;
+ @JsonProperty("project_name")
+ private String projectName;
+ @JsonProperty("endpoint_name")
+ private String endpointName;
+ @JsonProperty("grafana_admin")
+ private String grafanaAdmin;
+ @JsonProperty("grafana_pass")
+ private String grafanaPassword;
+ @JsonProperty("oauth_cookie_secret")
+ private String oauthCookieSecret;
+ @JsonProperty("odahuflow_connection_decrypt_token")
+ private String decryptToken;
+ @JsonProperty("odahu_urls")
+ private List<ResourceURL> resourceUrls;
+}
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/odahu/ActionOdahuDTO.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/odahu/ActionOdahuDTO.java
new file mode 100644
index 0000000..544ba82
--- /dev/null
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/odahu/ActionOdahuDTO.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.dto.odahu;
+
+import com.epam.dlab.dto.ResourceBaseDTO;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class ActionOdahuDTO extends ResourceBaseDTO<ActionOdahuDTO> {
+ @JsonProperty("odahu_cluster_name")
+ private final String name;
+ @JsonProperty("project_name")
+ private final String project;
+ @JsonProperty("endpoint_name")
+ private final String endpoint;
+ @JsonProperty("ssh_key")
+ private final String key;
+ @JsonProperty("grafana_admin")
+ private String grafanaAdmin;
+ @JsonProperty("grafana_pass")
+ private String grafanaPassword;
+ @JsonProperty("oauth_cookie_secret")
+ private String oauthCookieSecret;
+ @JsonProperty("odahuflow_connection_decrypt_token")
+ private String decryptToken;
+}
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/dto/odahu/CreateOdahuDTO.java b/services/dlab-model/src/main/java/com/epam/dlab/dto/odahu/CreateOdahuDTO.java
new file mode 100644
index 0000000..3fea2d4
--- /dev/null
+++ b/services/dlab-model/src/main/java/com/epam/dlab/dto/odahu/CreateOdahuDTO.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.dto.odahu;
+
+import com.epam.dlab.dto.ResourceBaseDTO;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class CreateOdahuDTO extends ResourceBaseDTO<CreateOdahuDTO> {
+ @JsonProperty("odahu_cluster_name")
+ private final String name;
+ @JsonProperty("project_name")
+ private final String project;
+ @JsonProperty("endpoint_name")
+ private final String endpoint;
+ @JsonProperty("ssh_key")
+ private final String key;
+}
diff --git a/services/dlab-model/src/main/java/com/epam/dlab/model/ResourceEnum.java b/services/dlab-model/src/main/java/com/epam/dlab/model/ResourceEnum.java
index 4ee7d2c..dd96c8f 100644
--- a/services/dlab-model/src/main/java/com/epam/dlab/model/ResourceEnum.java
+++ b/services/dlab-model/src/main/java/com/epam/dlab/model/ResourceEnum.java
@@ -22,7 +22,8 @@
public enum ResourceEnum {
EDGE_NODE("edge node"),
- NOTEBOOK("notebook");
+ NOTEBOOK("notebook"),
+ ODAHU("odahu");
private String name;
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/ProvisioningServiceApplication.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/ProvisioningServiceApplication.java
index 6f1047b..0808934 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/ProvisioningServiceApplication.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/ProvisioningServiceApplication.java
@@ -144,6 +144,7 @@
jersey.register(injector.getInstance(KeyResource.class));
jersey.register(injector.getInstance(CallbackHandlerResource.class));
jersey.register(injector.getInstance(ProjectResource.class));
+ jersey.register(injector.getInstance(OdahuResource.class));
jersey.register(injector.getInstance(ProvisioningHealthCheckResource.class));
}
}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/commands/CommandExecutorMockAsync.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/commands/CommandExecutorMockAsync.java
index 7277961..dae3b78 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/commands/CommandExecutorMockAsync.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/commands/CommandExecutorMockAsync.java
@@ -294,7 +294,7 @@
private void action(String user, DockerAction action) {
String resourceType = parser.getResourceType();
- String prefixFileName = (Lists.newArrayList("project", "edge", "dataengine", "dataengine-service")
+ String prefixFileName = (Lists.newArrayList("project", "edge", "odahu", "dataengine", "dataengine-service")
.contains(resourceType) ? resourceType : "notebook") + "_";
String templateFileName = "mock_response/" + cloudProvider.getName() + '/' + prefixFileName +
action.toString() + JSON_FILE_ENDING;
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/response/handlers/OdahuCallbackHandler.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/response/handlers/OdahuCallbackHandler.java
new file mode 100644
index 0000000..1c63819
--- /dev/null
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/core/response/handlers/OdahuCallbackHandler.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.core.response.handlers;
+
+import com.epam.dlab.backendapi.core.commands.DockerAction;
+import com.epam.dlab.dto.ResourceURL;
+import com.epam.dlab.dto.base.odahu.OdahuResult;
+import com.epam.dlab.rest.client.RESTService;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.extern.slf4j.Slf4j;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+@Slf4j
+public class OdahuCallbackHandler extends ResourceCallbackHandler<OdahuResult> {
+
+ private static final String ODAHU_URLS_FIELD = "odahu_urls";
+ private static final String GRAFANA_ADMIN = "grafana_admin";
+ private static final String GRAFANA_PASSWORD = "grafana_pass";
+ private static final String OAUTH_COOKIE_SECRET = "oauth_cookie_secret";
+ private static final String DECRYPT_TOKEN = "odahuflow_connection_decrypt_token";
+ private final String callbackUri;
+ private final String name;
+ private final String projectName;
+ private final String endpointName;
+
+ public OdahuCallbackHandler(RESTService selfService, String user, String uuid, DockerAction action,
+ String callbackUri, String name, String projectName, String endpointName) {
+ super(selfService, user, uuid, action);
+ this.callbackUri = callbackUri;
+ this.name = name;
+ this.projectName = projectName;
+ this.endpointName = endpointName;
+ }
+
+ @Override
+ protected String getCallbackURI() {
+ return callbackUri;
+ }
+
+ @Override
+ protected OdahuResult parseOutResponse(JsonNode resultNode, OdahuResult result) {
+ result.setName(name);
+ result.setProjectName(projectName);
+ result.setEndpointName(endpointName);
+
+ if (resultNode == null) {
+ return result;
+ }
+ result.setGrafanaAdmin(getTextValue(resultNode.get(GRAFANA_ADMIN)));
+ result.setGrafanaPassword(getTextValue(resultNode.get(GRAFANA_PASSWORD)));
+ result.setOauthCookieSecret(getTextValue(resultNode.get(OAUTH_COOKIE_SECRET)));
+ result.setDecryptToken(getTextValue(resultNode.get(DECRYPT_TOKEN)));
+
+ final JsonNode odahuUrls = resultNode.get(ODAHU_URLS_FIELD);
+ List<ResourceURL> urls = null;
+ if (odahuUrls != null) {
+ try {
+ urls = mapper.readValue(odahuUrls.toString(), new TypeReference<List<ResourceURL>>() {
+ });
+ result.setResourceUrls(urls);
+ } catch (IOException e) {
+ log.warn("Cannot parse field {} for UUID {} in JSON",
+ RESPONSE_NODE + "." + RESULT_NODE + "." + ODAHU_URLS_FIELD, getUUID(), e);
+ }
+ }
+
+ if (getAction() == DockerAction.CREATE && Objects.isNull(urls)) {
+ log.warn("There are no odahu urls in response file while creating {}", result);
+ }
+
+ return result;
+ }
+}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
index 40744fa..f510ec8 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
@@ -28,9 +28,11 @@
import com.epam.dlab.backendapi.core.response.handlers.dao.CallbackHandlerDao;
import com.epam.dlab.backendapi.core.response.handlers.dao.FileSystemCallbackHandlerDao;
import com.epam.dlab.backendapi.service.CheckInactivityService;
+import com.epam.dlab.backendapi.service.OdahuService;
import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.backendapi.service.RestoreCallbackHandlerService;
import com.epam.dlab.backendapi.service.impl.CheckInactivityServiceImpl;
+import com.epam.dlab.backendapi.service.impl.OdahuServiceImpl;
import com.epam.dlab.backendapi.service.impl.ProjectServiceImpl;
import com.epam.dlab.backendapi.service.impl.RestoreCallbackHandlerServiceImpl;
import com.epam.dlab.constants.ServiceConsts;
@@ -73,5 +75,6 @@
bind(RestoreCallbackHandlerService.class).to(RestoreCallbackHandlerServiceImpl.class);
bind(CheckInactivityService.class).to(CheckInactivityServiceImpl.class);
bind(ProjectService.class).to(ProjectServiceImpl.class);
+ bind(OdahuService.class).to(OdahuServiceImpl.class);
}
}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProvisioningDevModule.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProvisioningDevModule.java
index 73d333f..8329c71 100644
--- a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProvisioningDevModule.java
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/modules/ProvisioningDevModule.java
@@ -30,11 +30,12 @@
import com.epam.dlab.backendapi.core.commands.ICommandExecutor;
import com.epam.dlab.backendapi.core.response.handlers.dao.CallbackHandlerDao;
import com.epam.dlab.backendapi.core.response.handlers.dao.FileSystemCallbackHandlerDao;
+import com.epam.dlab.backendapi.service.CheckInactivityService;
+import com.epam.dlab.backendapi.service.OdahuService;
import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.backendapi.service.RestoreCallbackHandlerService;
-import com.epam.dlab.backendapi.service.CheckInactivityService;
-import com.epam.dlab.backendapi.service.RestoreCallbackHandlerService;
import com.epam.dlab.backendapi.service.impl.CheckInactivityServiceImpl;
+import com.epam.dlab.backendapi.service.impl.OdahuServiceImpl;
import com.epam.dlab.backendapi.service.impl.ProjectServiceImpl;
import com.epam.dlab.backendapi.service.impl.RestoreCallbackHandlerServiceImpl;
import com.epam.dlab.constants.ServiceConsts;
@@ -80,6 +81,7 @@
bind(RestoreCallbackHandlerService.class).to(RestoreCallbackHandlerServiceImpl.class);
bind(CheckInactivityService.class).to(CheckInactivityServiceImpl.class);
bind(ProjectService.class).to(ProjectServiceImpl.class);
+ bind(OdahuService.class).to(OdahuServiceImpl.class);
}
/**
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/OdahuResource.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/OdahuResource.java
new file mode 100644
index 0000000..670a653
--- /dev/null
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/resources/OdahuResource.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.resources;
+
+import com.epam.dlab.auth.UserInfo;
+import com.epam.dlab.backendapi.service.OdahuService;
+import com.epam.dlab.dto.odahu.ActionOdahuDTO;
+import com.epam.dlab.dto.odahu.CreateOdahuDTO;
+import com.google.inject.Inject;
+import io.dropwizard.auth.Auth;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+@Path("infrastructure/odahu")
+@Consumes(MediaType.APPLICATION_JSON)
+@Produces(MediaType.APPLICATION_JSON)
+public class OdahuResource {
+
+ private final OdahuService odahuService;
+
+ @Inject
+ public OdahuResource(OdahuService odahuService) {
+ this.odahuService = odahuService;
+ }
+
+ @POST
+ public Response createProject(@Auth UserInfo userInfo, CreateOdahuDTO dto) {
+ return Response.ok(odahuService.create(userInfo, dto)).build();
+ }
+
+ @Path("start")
+ @POST
+ public Response startProject(@Auth UserInfo userInfo, ActionOdahuDTO dto) {
+ return Response.ok(odahuService.start(userInfo, dto)).build();
+ }
+
+ @Path("stop")
+ @POST
+ public Response stopProject(@Auth UserInfo userInfo, ActionOdahuDTO dto) {
+ return Response.ok(odahuService.stop(userInfo, dto)).build();
+ }
+
+ @Path("terminate")
+ @POST
+ public Response terminateProject(@Auth UserInfo userInfo, ActionOdahuDTO dto) {
+ return Response.ok(odahuService.terminate(userInfo, dto)).build();
+ }
+}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/OdahuService.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/OdahuService.java
new file mode 100644
index 0000000..286e2be
--- /dev/null
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/OdahuService.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.service;
+
+import com.epam.dlab.auth.UserInfo;
+import com.epam.dlab.dto.odahu.ActionOdahuDTO;
+import com.epam.dlab.dto.odahu.CreateOdahuDTO;
+
+public interface OdahuService {
+ String create(UserInfo userInfo, CreateOdahuDTO odahuCreateDTO);
+
+ String start(UserInfo userInfo, ActionOdahuDTO dto);
+
+ String stop(UserInfo userInfo, ActionOdahuDTO dto);
+
+ String terminate(UserInfo userInfo, ActionOdahuDTO dto);
+}
diff --git a/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/OdahuServiceImpl.java b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/OdahuServiceImpl.java
new file mode 100644
index 0000000..5d032e0
--- /dev/null
+++ b/services/provisioning-service/src/main/java/com/epam/dlab/backendapi/service/impl/OdahuServiceImpl.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.service.impl;
+
+import com.epam.dlab.auth.UserInfo;
+import com.epam.dlab.backendapi.ProvisioningServiceApplicationConfiguration;
+import com.epam.dlab.backendapi.core.commands.CommandBuilder;
+import com.epam.dlab.backendapi.core.commands.DockerAction;
+import com.epam.dlab.backendapi.core.commands.DockerCommands;
+import com.epam.dlab.backendapi.core.commands.ICommandExecutor;
+import com.epam.dlab.backendapi.core.commands.RunDockerCommand;
+import com.epam.dlab.backendapi.core.response.folderlistener.FolderListenerExecutor;
+import com.epam.dlab.backendapi.core.response.handlers.OdahuCallbackHandler;
+import com.epam.dlab.backendapi.service.OdahuService;
+import com.epam.dlab.dto.ResourceBaseDTO;
+import com.epam.dlab.dto.odahu.ActionOdahuDTO;
+import com.epam.dlab.dto.odahu.CreateOdahuDTO;
+import com.epam.dlab.rest.client.RESTService;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.inject.Inject;
+
+public class OdahuServiceImpl implements OdahuService {
+
+ private static final String CALLBACK_URI = "/api/odahu/status";
+ private static final String ODAHU_RESOURCE_TYPE = "odahu";
+ private static final String ODAHU_IMAGE = "docker.dlab-odahu";
+
+ private final ProvisioningServiceApplicationConfiguration configuration;
+ private final FolderListenerExecutor folderListenerExecutor;
+ private final CommandBuilder commandBuilder;
+ private final ICommandExecutor commandExecutor;
+ private final RESTService selfService;
+
+ @Inject
+ public OdahuServiceImpl(ProvisioningServiceApplicationConfiguration configuration,
+ FolderListenerExecutor folderListenerExecutor, CommandBuilder commandBuilder,
+ ICommandExecutor commandExecutor, RESTService selfService) {
+ this.configuration = configuration;
+ this.folderListenerExecutor = folderListenerExecutor;
+ this.commandBuilder = commandBuilder;
+ this.commandExecutor = commandExecutor;
+ this.selfService = selfService;
+ }
+
+ @Override
+ public String create(UserInfo userInfo, CreateOdahuDTO dto) {
+ return executeDocker(userInfo, dto, DockerAction.CREATE, ODAHU_RESOURCE_TYPE, ODAHU_IMAGE, dto.getName(),
+ dto.getProject(), dto.getEndpoint());
+ }
+
+ @Override
+ public String start(UserInfo userInfo, ActionOdahuDTO dto) {
+ return executeDocker(userInfo, dto, DockerAction.START, ODAHU_RESOURCE_TYPE, ODAHU_IMAGE, dto.getName(),
+ dto.getProject(), dto.getEndpoint());
+ }
+
+ @Override
+ public String stop(UserInfo userInfo, ActionOdahuDTO dto) {
+ return executeDocker(userInfo, dto, DockerAction.STOP, ODAHU_RESOURCE_TYPE, ODAHU_IMAGE, dto.getName(),
+ dto.getProject(), dto.getEndpoint());
+ }
+
+ @Override
+ public String terminate(UserInfo userInfo, ActionOdahuDTO dto) {
+ return executeDocker(userInfo, dto, DockerAction.TERMINATE, ODAHU_RESOURCE_TYPE, ODAHU_IMAGE, dto.getName(),
+ dto.getProject(), dto.getEndpoint());
+ }
+
+
+ private String executeDocker(UserInfo userInfo, ResourceBaseDTO dto, DockerAction action, String resourceType,
+ String image, String name, String project, String endpoint) {
+ String uuid = DockerCommands.generateUUID();
+
+ folderListenerExecutor.start(configuration.getKeyLoaderDirectory(),
+ configuration.getKeyLoaderPollTimeout(),
+ new OdahuCallbackHandler(selfService, userInfo.getName(), uuid, action, CALLBACK_URI, name, project, endpoint));
+
+ RunDockerCommand runDockerCommand = new RunDockerCommand()
+ .withInteractive()
+ .withName(String.join("_", userInfo.getSimpleName(), name, resourceType, action.toString(),
+ Long.toString(System.currentTimeMillis())))
+ .withVolumeForRootKeys(configuration.getKeyDirectory())
+ .withVolumeForResponse(configuration.getKeyLoaderDirectory())
+ .withVolumeForLog(configuration.getDockerLogDirectory(), resourceType)
+ .withResource(resourceType)
+ .withRequestId(uuid)
+ .withConfKeyName(configuration.getAdminKey())
+ .withImage(image)
+ .withAction(action);
+
+ try {
+ commandExecutor.executeAsync(userInfo.getName(), uuid, commandBuilder.buildCommand(runDockerCommand, dto));
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+ return uuid;
+ }
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/aws/odahu_create.json b/services/provisioning-service/src/main/resources/mock_response/aws/odahu_create.json
new file mode 100644
index 0000000..955d08b
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/aws/odahu_create.json
@@ -0,0 +1,23 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ "odahu_urls": [
+ {
+ "url": "http://10.10.16.2:8888/",
+ "description": "API Gateway"
+ },
+ {
+ "url": "http://10.10.16.2:8085/",
+ "description": "Documentation"
+ }
+ ],
+ "oauth_cookie_secret": "WlFRZzhhZTJxbTFNa0JpUw==",
+ "odahuflow_connection_decrypt_token": "qfY2k5bA2M",
+ "grafana_pass": "7h0uum1EGe",
+ "grafana_admin": "grafana_admin"
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/aws/odahu_start.json b/services/provisioning-service/src/main/resources/mock_response/aws/odahu_start.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/aws/odahu_start.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/aws/odahu_stop.json b/services/provisioning-service/src/main/resources/mock_response/aws/odahu_stop.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/aws/odahu_stop.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/aws/odahu_terminate.json b/services/provisioning-service/src/main/resources/mock_response/aws/odahu_terminate.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/aws/odahu_terminate.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/azure/odahu_create.json b/services/provisioning-service/src/main/resources/mock_response/azure/odahu_create.json
new file mode 100644
index 0000000..955d08b
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/azure/odahu_create.json
@@ -0,0 +1,23 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ "odahu_urls": [
+ {
+ "url": "http://10.10.16.2:8888/",
+ "description": "API Gateway"
+ },
+ {
+ "url": "http://10.10.16.2:8085/",
+ "description": "Documentation"
+ }
+ ],
+ "oauth_cookie_secret": "WlFRZzhhZTJxbTFNa0JpUw==",
+ "odahuflow_connection_decrypt_token": "qfY2k5bA2M",
+ "grafana_pass": "7h0uum1EGe",
+ "grafana_admin": "grafana_admin"
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/azure/odahu_start.json b/services/provisioning-service/src/main/resources/mock_response/azure/odahu_start.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/azure/odahu_start.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/azure/odahu_stop.json b/services/provisioning-service/src/main/resources/mock_response/azure/odahu_stop.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/azure/odahu_stop.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/azure/odahu_terminate.json b/services/provisioning-service/src/main/resources/mock_response/azure/odahu_terminate.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/azure/odahu_terminate.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_create.json b/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_create.json
new file mode 100644
index 0000000..955d08b
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_create.json
@@ -0,0 +1,23 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ "odahu_urls": [
+ {
+ "url": "http://10.10.16.2:8888/",
+ "description": "API Gateway"
+ },
+ {
+ "url": "http://10.10.16.2:8085/",
+ "description": "Documentation"
+ }
+ ],
+ "oauth_cookie_secret": "WlFRZzhhZTJxbTFNa0JpUw==",
+ "odahuflow_connection_decrypt_token": "qfY2k5bA2M",
+ "grafana_pass": "7h0uum1EGe",
+ "grafana_admin": "grafana_admin"
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_start.json b/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_start.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_start.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_stop.json b/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_stop.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_stop.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_terminate.json b/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_terminate.json
new file mode 100644
index 0000000..99433bd
--- /dev/null
+++ b/services/provisioning-service/src/main/resources/mock_response/gcp/odahu_terminate.json
@@ -0,0 +1,9 @@
+{
+ "status": "ok",
+ "response": {
+ "result": {
+ },
+ "log": "/var/log/dlab/odahu/odahu_${EDGE_USER_NAME}_${REQUEST_ID}.log"
+ },
+ "request_id": "${REQUEST_ID}"
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/SelfServiceApplication.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/SelfServiceApplication.java
index 6a12ea5..3b268b9 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/SelfServiceApplication.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/SelfServiceApplication.java
@@ -161,6 +161,8 @@
jersey.register(injector.getInstance(EndpointResource.class));
jersey.register(injector.getInstance(ProjectResource.class));
jersey.register(injector.getInstance(ProjectCallback.class));
+ jersey.register(injector.getInstance(OdahuResource.class));
+ jersey.register(injector.getInstance(OdahuCallback.class));
}
private void disableGzipHandlerForGuacamoleServlet(Server server) {
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java
index ebe81c5..1adade7 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/ExploratoryDAO.java
@@ -113,8 +113,8 @@
* @param user name
* @return list of user resources
*/
- public Iterable<Document> findExploratory(String user) {
- return find(USER_INSTANCES, eq(USER, user),
+ public Iterable<Document> findExploratory(String user, String project) {
+ return find(USER_INSTANCES, and(eq(USER, user), eq(PROJECT, project)),
fields(exclude(ExploratoryLibDAO.EXPLORATORY_LIBS, ExploratoryLibDAO.COMPUTATIONAL_LIBS, SCHEDULER_DATA,
EXPLORATORY_USER, EXPLORATORY_PASS)));
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/OdahuDAO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/OdahuDAO.java
new file mode 100644
index 0000000..f323dbc
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/OdahuDAO.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.dao;
+
+import com.epam.dlab.backendapi.domain.OdahuDTO;
+import com.epam.dlab.backendapi.domain.OdahuFieldsDTO;
+import com.epam.dlab.dto.UserInstanceStatus;
+import com.epam.dlab.dto.base.odahu.OdahuResult;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface OdahuDAO {
+ Optional<OdahuDTO> getByProjectEndpoint(String project, String endpoint);
+
+ OdahuFieldsDTO getFields(String name, String project, String endpoint);
+
+ List<OdahuDTO> findOdahuClusters();
+
+ boolean create(OdahuDTO odahuDTO);
+
+ void updateStatus(String name, String project, String endpoint, UserInstanceStatus status);
+
+ void updateStatusAndUrls(OdahuResult result, UserInstanceStatus status);
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/OdahuDAOImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/OdahuDAOImpl.java
new file mode 100644
index 0000000..5ba4c80
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/dao/OdahuDAOImpl.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.dao;
+
+import com.epam.dlab.backendapi.domain.OdahuDTO;
+import com.epam.dlab.backendapi.domain.OdahuFieldsDTO;
+import com.epam.dlab.backendapi.domain.ProjectDTO;
+import com.epam.dlab.dto.ResourceURL;
+import com.epam.dlab.dto.UserInstanceStatus;
+import com.epam.dlab.dto.base.odahu.OdahuResult;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.mongodb.BasicDBObject;
+import com.mongodb.client.result.UpdateResult;
+import org.bson.Document;
+import org.bson.conversions.Bson;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static com.mongodb.client.model.Filters.and;
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Projections.elemMatch;
+import static com.mongodb.client.model.Projections.excludeId;
+import static com.mongodb.client.model.Projections.fields;
+import static com.mongodb.client.model.Projections.include;
+import static com.mongodb.client.model.Updates.push;
+import static java.util.stream.Collectors.toList;
+
+public class OdahuDAOImpl extends BaseDAO implements OdahuDAO {
+
+ private static final String PROJECTS_COLLECTION = "Projects";
+ private static final String ENDPOINTS = "endpoints";
+ private static final String ODAHU_FIELD = "odahu";
+ private static final String NAME_FIELD = "name";
+ private static final String ENDPOINT_FIELD = "endpoint";
+ private static final String PROJECT_FIELD = "project";
+ private static final String STATUS_FIELD = "status";
+ private static final String GRAFANA_ADMIN_FIELD = "grafana_admin";
+ private static final String GRAFANA_PASSWORD_FIELD = "grafana_pass";
+ private static final String OAUTH_COOKIE_SECRET_FIELD = "oauth_cookie_secret";
+ private static final String DECRYPT_TOKEN_FIELD = "odahuflow_connection_decrypt_token";
+ private static final String URLS_FIELD = "urls";
+ private static final String COMPUTATIONAL_URL_DESC = "description";
+ private static final String COMPUTATIONAL_URL_URL = "url";
+
+ @Override
+ public Optional<OdahuDTO> getByProjectEndpoint(String project, String endpoint) {
+ Optional<ProjectDTO> projectDTO = findOne(PROJECTS_COLLECTION, odahuProjectEndpointCondition(project, endpoint),
+ fields(include(ODAHU_FIELD), excludeId()),
+ ProjectDTO.class);
+
+ return projectDTO.flatMap(p -> p.getOdahu()
+ .stream()
+ .filter(odahu -> project.equals(odahu.getProject()) && endpoint.equals(odahu.getEndpoint()))
+ .findAny());
+ }
+
+ @Override
+ public OdahuFieldsDTO getFields(String name, String project, String endpoint) {
+ Optional<Document> one = findOne(PROJECTS_COLLECTION, odahuProjectEndpointCondition(name, project, endpoint),
+ fields(include(ODAHU_FIELD), excludeId()));
+ OdahuFieldsDTO odahuFields = null;
+ if (one.isPresent()) {
+ List<OdahuFieldsDTO> list = convertFromDocument(one.get().get(ODAHU_FIELD, ArrayList.class), new TypeReference<List<OdahuFieldsDTO>>() {
+ });
+ odahuFields = list.get(0);
+ }
+ return odahuFields;
+ }
+
+ @Override
+ public List<OdahuDTO> findOdahuClusters() {
+ List<ProjectDTO> projectDTOS = find(PROJECTS_COLLECTION, ProjectDTO.class);
+ return projectDTOS.stream()
+ .map(ProjectDTO::getOdahu)
+ .flatMap(List::stream)
+ .collect(toList());
+ }
+
+ @Override
+ public boolean create(OdahuDTO odahuDTO) {
+ UpdateResult updateResult = updateOne(PROJECTS_COLLECTION, projectEndpointCondition(odahuDTO.getProject(),
+ odahuDTO.getEndpoint()),
+ push(ODAHU_FIELD, convertToBson(odahuDTO)));
+ return updateResult.getModifiedCount() > 0;
+ }
+
+ @Override
+ public void updateStatus(String name, String project, String endpoint, UserInstanceStatus status) {
+ BasicDBObject dbObject = new BasicDBObject();
+ dbObject.put(ODAHU_FIELD + ".$." + STATUS_FIELD, status.name());
+ updateOne(PROJECTS_COLLECTION, and(elemMatch(ODAHU_FIELD, eq(NAME_FIELD, name)),
+ odahuProjectEndpointCondition(project, endpoint)), new Document(SET, dbObject));
+ }
+
+ @Override
+ public void updateStatusAndUrls(OdahuResult result, UserInstanceStatus status) {
+ BasicDBObject dbObject = new BasicDBObject();
+ dbObject.put(ODAHU_FIELD + ".$." + STATUS_FIELD, status.name());
+ dbObject.put(ODAHU_FIELD + ".$." + URLS_FIELD, getResourceUrlData(result.getResourceUrls()));
+ dbObject.put(ODAHU_FIELD + ".$." + GRAFANA_ADMIN_FIELD, result.getGrafanaAdmin());
+ dbObject.put(ODAHU_FIELD + ".$." + GRAFANA_PASSWORD_FIELD, result.getGrafanaPassword());
+ dbObject.put(ODAHU_FIELD + ".$." + OAUTH_COOKIE_SECRET_FIELD, result.getOauthCookieSecret());
+ dbObject.put(ODAHU_FIELD + ".$." + DECRYPT_TOKEN_FIELD, result.getDecryptToken());
+ updateOne(PROJECTS_COLLECTION, odahuProjectEndpointCondition(result.getName(), result.getProjectName(), result.getEndpointName()),
+ new Document(SET, dbObject));
+ }
+
+ private Bson odahuProjectEndpointCondition(String name, String projectName, String endpointName) {
+ return and(elemMatch(ODAHU_FIELD, eq(NAME_FIELD, name)), odahuProjectEndpointCondition(projectName, endpointName));
+ }
+
+ private Bson odahuProjectEndpointCondition(String projectName, String endpointName) {
+ return elemMatch(ODAHU_FIELD, and(eq(ENDPOINT_FIELD, endpointName), eq(PROJECT_FIELD, projectName)));
+ }
+
+ private Bson projectEndpointCondition(String projectName, String endpointName) {
+ return and(eq(NAME_FIELD, projectName), and(elemMatch(ENDPOINTS, eq(NAME_FIELD, endpointName))));
+ }
+
+ private List<Map<String, String>> getResourceUrlData(List<ResourceURL> urls) {
+ return urls.stream()
+ .map(this::toUrlDocument)
+ .collect(toList());
+ }
+
+ private LinkedHashMap<String, String> toUrlDocument(ResourceURL url) {
+ LinkedHashMap<String, String> map = new LinkedHashMap<>();
+ map.put(COMPUTATIONAL_URL_URL, url.getUrl());
+ map.put(COMPUTATIONAL_URL_DESC, url.getDescription());
+ return map;
+ }
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuActionDTO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuActionDTO.java
new file mode 100644
index 0000000..fa86750
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuActionDTO.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.domain;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import lombok.Data;
+
+import javax.validation.constraints.NotNull;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class OdahuActionDTO {
+ @NotNull
+ private final String name;
+ @NotNull
+ private final String project;
+ @NotNull
+ private final String endpoint;
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuCreateDTO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuCreateDTO.java
new file mode 100644
index 0000000..6c6cb27
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuCreateDTO.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.domain;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+import javax.validation.constraints.NotNull;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class OdahuCreateDTO {
+ @NotNull
+ private final String name;
+ @NotNull
+ private final String project;
+ @NotNull
+ private final String endpoint;
+ @JsonProperty("custom_tag")
+ private final String customTag;
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuDTO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuDTO.java
new file mode 100644
index 0000000..6ecf35c
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuDTO.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.domain;
+
+import com.epam.dlab.dto.ResourceURL;
+import com.epam.dlab.dto.UserInstanceStatus;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class OdahuDTO {
+ private final String name;
+ private final String project;
+ private final String endpoint;
+ @JsonProperty("grafana_admin")
+ private String grafanaAdmin;
+ @JsonProperty("grafana_pass")
+ private String grafanaPassword;
+ private final List<ResourceURL> urls = new ArrayList<>();
+ private final UserInstanceStatus status;
+ private final Map<String, String> tags;
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuFieldsDTO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuFieldsDTO.java
new file mode 100644
index 0000000..7a71795
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/OdahuFieldsDTO.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.domain;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class OdahuFieldsDTO {
+ @JsonProperty("grafana_admin")
+ private String grafanaAdmin;
+ @JsonProperty("grafana_pass")
+ private String grafanaPassword;
+ @JsonProperty("oauth_cookie_secret")
+ private String oauthCookieSecret;
+ @JsonProperty("odahuflow_connection_decrypt_token")
+ private String decryptToken;
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/ProjectDTO.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/ProjectDTO.java
index 9cd0a35..6bd8a1e 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/ProjectDTO.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/domain/ProjectDTO.java
@@ -8,6 +8,7 @@
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Pattern;
+import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -28,6 +29,7 @@
private final Integer budget;
private final List<ProjectEndpointDTO> endpoints;
private final boolean sharedImageEnabled;
+ private final List<OdahuDTO> odahu = new ArrayList<>();
public enum Status {
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/DevModule.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/DevModule.java
index cf08d12..f1a9a3e 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/DevModule.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/DevModule.java
@@ -106,6 +106,8 @@
bind(EndpointDAO.class).to(EndpointDAOImpl.class);
bind(ProjectService.class).to(ProjectServiceImpl.class);
bind(ProjectDAO.class).to(ProjectDAOImpl.class);
+ bind(OdahuDAO.class).to(OdahuDAOImpl.class);
+ bind(OdahuService.class).to(OdahuServiceImpl.class);
}
private void configureCors(Environment environment) {
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
index 0d0ae1d..b67584e 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/modules/ProductionModule.java
@@ -99,5 +99,7 @@
bind(SecurityService.class).to(SecurityServiceImpl.class);
bind(KeycloakService.class).to(KeycloakServiceImpl.class);
bind(Client.class).toInstance(httpClient);
+ bind(OdahuDAO.class).to(OdahuDAOImpl.class);
+ bind(OdahuService.class).to(OdahuServiceImpl.class);
}
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResource.java
index db8197f..b9dee92 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResource.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResource.java
@@ -78,7 +78,7 @@
@GET
@Path("/info")
public List<ProjectInfrastructureInfo> getUserResources(@Auth UserInfo userInfo) {
- return infrastructureInfoService.getUserResources(userInfo.getName());
+ return infrastructureInfoService.getUserResources(userInfo);
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/OdahuResource.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/OdahuResource.java
new file mode 100644
index 0000000..a254b9b
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/OdahuResource.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.resources;
+
+import com.epam.dlab.auth.UserInfo;
+import com.epam.dlab.backendapi.domain.OdahuActionDTO;
+import com.epam.dlab.backendapi.domain.OdahuCreateDTO;
+import com.epam.dlab.backendapi.service.OdahuService;
+import com.google.inject.Inject;
+import io.dropwizard.auth.Auth;
+import io.swagger.v3.oas.annotations.Parameter;
+
+import javax.annotation.security.RolesAllowed;
+import javax.validation.Valid;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriInfo;
+import java.net.URI;
+
+@Path("odahu")
+@Consumes(MediaType.APPLICATION_JSON)
+public class OdahuResource {
+
+ private final OdahuService odahuService;
+
+ @Inject
+ public OdahuResource(OdahuService odahuService) {
+ this.odahuService = odahuService;
+ }
+
+ @GET
+ @RolesAllowed("/api/odahu")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Response getOdahuClusters(@Parameter(hidden = true) @Auth UserInfo userInfo) {
+ return Response.ok(odahuService.findOdahu()).build();
+ }
+
+ @POST
+ @RolesAllowed("/api/odahu")
+ public Response createOdahuCluster(@Parameter(hidden = true) @Auth UserInfo userInfo,
+ @Parameter(hidden = true) @Context UriInfo uriInfo,
+ @Valid OdahuCreateDTO odahuCreateDTO) {
+ odahuService.create(odahuCreateDTO.getProject(), odahuCreateDTO, userInfo);
+ final URI uri = uriInfo.getRequestUriBuilder().path(odahuCreateDTO.getName()).build();
+ return Response.created(uri).build();
+ }
+
+ @Path("start")
+ @POST
+ @RolesAllowed("/api/odahu")
+ public Response startOdahuCluster(@Parameter(hidden = true) @Auth UserInfo userInfo,
+ @Valid OdahuActionDTO startOdahuDTO) {
+ odahuService.start(startOdahuDTO.getName(), startOdahuDTO.getProject(), startOdahuDTO.getEndpoint(), userInfo);
+ return Response.accepted().build();
+ }
+
+ @Path("stop")
+ @POST
+ @RolesAllowed("/api/odahu")
+ public Response stopOdahuCluster(@Parameter(hidden = true) @Auth UserInfo userInfo,
+ @Valid OdahuActionDTO stopOdahuDTO) {
+ odahuService.stop(stopOdahuDTO.getName(), stopOdahuDTO.getProject(), stopOdahuDTO.getEndpoint(), userInfo);
+ return Response.accepted().build();
+ }
+
+ @Path("terminate")
+ @POST
+ @RolesAllowed("/api/odahu")
+ public Response terminateOdahuCluster(@Parameter(hidden = true) @Auth UserInfo userInfo,
+ @Valid OdahuActionDTO terminateOdahuDTO) {
+ odahuService.terminate(terminateOdahuDTO.getName(), terminateOdahuDTO.getProject(), terminateOdahuDTO.getEndpoint(), userInfo);
+ return Response.accepted().build();
+ }
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/callback/OdahuCallback.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/callback/OdahuCallback.java
new file mode 100644
index 0000000..6501ff7
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/callback/OdahuCallback.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.resources.callback;
+
+import com.epam.dlab.backendapi.domain.RequestId;
+import com.epam.dlab.backendapi.service.OdahuService;
+import com.epam.dlab.dto.UserInstanceStatus;
+import com.epam.dlab.dto.base.odahu.OdahuResult;
+import com.epam.dlab.exceptions.DlabException;
+import com.google.inject.Inject;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.util.Optional;
+
+@Path("odahu/status")
+@Consumes(MediaType.APPLICATION_JSON)
+public class OdahuCallback {
+
+ private final OdahuService odahuService;
+ private final RequestId requestId;
+
+ @Inject
+ public OdahuCallback(OdahuService odahuService, RequestId requestId) {
+ this.odahuService = odahuService;
+ this.requestId = requestId;
+ }
+
+ @POST
+ public Response updateOdahuStatus(OdahuResult result) {
+ requestId.checkAndRemove(result.getRequestId());
+ final UserInstanceStatus status = UserInstanceStatus.of(result.getStatus());
+ Optional.ofNullable(status)
+ .orElseThrow(() -> new DlabException(String.format("Cannot convert %s to UserInstanceStatus", status)));
+
+ odahuService.updateStatus(result, status);
+ return Response.ok().build();
+ }
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
index 44837a2..5a436ea 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/resources/dto/ProjectInfrastructureInfo.java
@@ -20,6 +20,7 @@
package com.epam.dlab.backendapi.resources.dto;
import com.epam.dlab.backendapi.domain.EndpointDTO;
+import com.epam.dlab.backendapi.domain.OdahuDTO;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.ToString;
@@ -40,5 +41,7 @@
@JsonProperty
private Iterable<Document> exploratory;
@JsonProperty
+ private List<OdahuDTO> odahu;
+ @JsonProperty
private List<EndpointDTO> endpoints;
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/InfrastructureInfoService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/InfrastructureInfoService.java
index aa23d1d..a385925 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/InfrastructureInfoService.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/InfrastructureInfoService.java
@@ -27,7 +27,7 @@
import java.util.List;
public interface InfrastructureInfoService {
- List<ProjectInfrastructureInfo> getUserResources(String user);
+ List<ProjectInfrastructureInfo> getUserResources(UserInfo user);
HealthStatusPageDTO getHeathStatus(UserInfo user, boolean fullReport, boolean isAdmin);
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/OdahuService.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/OdahuService.java
new file mode 100644
index 0000000..e787991
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/OdahuService.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.service;
+
+import com.epam.dlab.auth.UserInfo;
+import com.epam.dlab.backendapi.domain.OdahuCreateDTO;
+import com.epam.dlab.backendapi.domain.OdahuDTO;
+import com.epam.dlab.dto.UserInstanceStatus;
+import com.epam.dlab.dto.base.odahu.OdahuResult;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface OdahuService {
+ List<OdahuDTO> findOdahu();
+
+ Optional<OdahuDTO> get(String project, String endpoint);
+
+ void create(String project, OdahuCreateDTO createOdahuDTO, UserInfo userInfo);
+
+ void start(String name, String project, String endpoint, UserInfo user);
+
+ void stop(String name, String project, String endpoint, UserInfo user);
+
+ void terminate(String name, String project, String endpoint, UserInfo user);
+
+ void updateStatus(OdahuResult odahuResult, UserInstanceStatus status);
+
+ boolean inProgress(String project, String endpoint);
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
index 9c71f76..4a8956d 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EndpointServiceImpl.java
@@ -8,6 +8,7 @@
import com.epam.dlab.backendapi.domain.EndpointResourcesDTO;
import com.epam.dlab.backendapi.domain.ProjectDTO;
import com.epam.dlab.backendapi.service.EndpointService;
+import com.epam.dlab.backendapi.service.OdahuService;
import com.epam.dlab.backendapi.service.ProjectService;
import com.epam.dlab.cloud.CloudProvider;
import com.epam.dlab.constants.ServiceConsts;
@@ -37,17 +38,19 @@
private final ProjectService projectService;
private final ExploratoryDAO exploratoryDAO;
private final RESTService provisioningService;
+ private final OdahuService odahuService;
private final UserRoleDao userRoleDao;
@Inject
public EndpointServiceImpl(EndpointDAO endpointDAO, ProjectService projectService, ExploratoryDAO exploratoryDAO,
@Named(ServiceConsts.PROVISIONING_SERVICE_NAME) RESTService provisioningService,
- UserRoleDao userRoleDao) {
+ OdahuService odahuService, UserRoleDao userRoleDao) {
this.endpointDAO = endpointDAO;
this.projectService = projectService;
this.exploratoryDAO = exploratoryDAO;
this.provisioningService = provisioningService;
+ this.odahuService = odahuService;
this.userRoleDao = userRoleDao;
}
@@ -149,7 +152,8 @@
private void checkProjectEndpointResourcesStatuses(List<ProjectDTO> projects, String endpoint) {
boolean isTerminationEnabled = projects.stream().anyMatch(p ->
- !projectService.checkExploratoriesAndComputationalProgress(p.getName(), Collections.singletonList(endpoint)) ||
+ odahuService.inProgress(p.getName(), endpoint) ||
+ !projectService.checkExploratoriesAndComputationalProgress(p.getName(), Collections.singletonList(endpoint)) ||
p.getEndpoints().stream().anyMatch(e -> e.getName().equals(endpoint) &&
Arrays.asList(UserInstanceStatus.CREATING, UserInstanceStatus.STARTING, UserInstanceStatus.STOPPING,
UserInstanceStatus.TERMINATING).contains(e.getStatus())));
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EnvironmentServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EnvironmentServiceImpl.java
index 3dfaaae..f86fd72 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EnvironmentServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/EnvironmentServiceImpl.java
@@ -23,6 +23,7 @@
import com.epam.dlab.backendapi.dao.EnvDAO;
import com.epam.dlab.backendapi.dao.ExploratoryDAO;
import com.epam.dlab.backendapi.dao.UserSettingsDAO;
+import com.epam.dlab.backendapi.domain.OdahuDTO;
import com.epam.dlab.backendapi.domain.ProjectDTO;
import com.epam.dlab.backendapi.resources.dto.UserDTO;
import com.epam.dlab.backendapi.resources.dto.UserResourceInfo;
@@ -98,7 +99,8 @@
List<UserInstanceDTO> expList = exploratoryDAO.getInstances();
return projectService.getProjects()
.stream()
- .map(projectDTO -> getProjectEnv(projectDTO, expList)).flatMap(Collection::stream)
+ .map(projectDTO -> getProjectEnv(projectDTO, expList))
+ .flatMap(Collection::stream)
.collect(toList());
}
@@ -186,7 +188,12 @@
private List<UserResourceInfo> getProjectEnv(ProjectDTO projectDTO, List<UserInstanceDTO> allInstances) {
final Stream<UserResourceInfo> userResources = allInstances.stream()
- .filter(instance -> instance.getProject().equals(projectDTO.getName())).map(this::toUserResourceInfo);
+ .filter(instance -> instance.getProject().equals(projectDTO.getName()))
+ .map(this::toUserResourceInfo);
+
+ Stream<UserResourceInfo> odahuResources = projectDTO.getOdahu().stream()
+ .map(this::toUserResourceInfo);
+
if (projectDTO.getEndpoints() != null) {
final Stream<UserResourceInfo> edges = projectDTO.getEndpoints()
.stream()
@@ -194,7 +201,7 @@
.withResourceStatus(e.getStatus().toString())
.withProject(projectDTO.getName())
.withIp(e.getEdgeInfo() != null ? e.getEdgeInfo().getPublicIp() : null));
- return Stream.concat(edges, userResources)
+ return Stream.concat(edges, Stream.concat(odahuResources, userResources))
.collect(toList());
} else {
return userResources.collect(toList());
@@ -212,6 +219,14 @@
.withCloudProvider(userInstance.getCloudProvider());
}
+ private UserResourceInfo toUserResourceInfo(OdahuDTO odahuDTO) {
+ return new UserResourceInfo()
+ .withResourceType(ResourceEnum.ODAHU)
+ .withResourceName(odahuDTO.getName())
+ .withResourceStatus(odahuDTO.getStatus().toString())
+ .withProject(odahuDTO.getProject());
+ }
+
private void checkProjectResourceConditions(String project, String action) {
final List<UserInstanceDTO> userInstances = exploratoryDAO
.fetchProjectExploratoriesWhereStatusIn(project,
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
index ecfe6cf..953da3f 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/InfrastructureInfoServiceImpl.java
@@ -25,6 +25,7 @@
import com.epam.dlab.backendapi.dao.EnvDAO;
import com.epam.dlab.backendapi.dao.ExploratoryDAO;
import com.epam.dlab.backendapi.domain.EndpointDTO;
+import com.epam.dlab.backendapi.domain.ProjectDTO;
import com.epam.dlab.backendapi.domain.ProjectEndpointDTO;
import com.epam.dlab.backendapi.resources.dto.HealthStatusPageDTO;
import com.epam.dlab.backendapi.resources.dto.ProjectInfrastructureInfo;
@@ -40,14 +41,12 @@
import com.google.inject.Inject;
import com.jcabi.manifests.Manifests;
import lombok.extern.slf4j.Slf4j;
-import org.bson.Document;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
@Slf4j
public class InfrastructureInfoServiceImpl implements InfrastructureInfoService {
@@ -69,29 +68,14 @@
@Override
- public List<ProjectInfrastructureInfo> getUserResources(String user) {
+ public List<ProjectInfrastructureInfo> getUserResources(UserInfo user) {
log.debug("Loading list of provisioned resources for user {}", user);
try {
- Iterable<Document> documents = expDAO.findExploratory(user);
List<EndpointDTO> allEndpoints = endpointService.getEndpoints();
- return StreamSupport.stream(documents.spliterator(),
- false)
- .collect(Collectors.groupingBy(d -> d.getString("project")))
- .entrySet()
- .stream()
- .map(e -> {
- List<ProjectEndpointDTO> endpoints = projectService.get(e.getKey()).getEndpoints();
- List<EndpointDTO> endpointResult = allEndpoints.stream()
- .filter(endpoint -> endpoints.stream()
- .anyMatch(endpoint1 -> endpoint1.getName().equals(endpoint.getName())))
- .collect(Collectors.toList());
- final Map<String, Map<String, String>> projectEdges =
- endpoints.stream()
- .collect(Collectors.toMap(ProjectEndpointDTO::getName,
- endpointDTO -> getSharedInfo(endpointDTO.getEdgeInfo())));
- return new ProjectInfrastructureInfo(e.getKey(),
- billingDAO.getBillingProjectQuoteUsed(e.getKey()), projectEdges, e.getValue(), endpointResult);
- })
+ return projectService.getUserProjects(user, false).stream()
+ .map(p -> new ProjectInfrastructureInfo(p.getName(), billingDAO.getBillingProjectQuoteUsed(p.getName()),
+ getSharedInfo(p.getName()), expDAO.findExploratory(user.getName(), p.getName()), p.getOdahu(),
+ getEndpoints(allEndpoints, p)))
.collect(Collectors.toList());
} catch (Exception e) {
log.error("Could not load list of provisioned resources for user: {}", user, e);
@@ -128,6 +112,18 @@
.build();
}
+ private List<EndpointDTO> getEndpoints(List<EndpointDTO> allEndpoints, ProjectDTO projectDTO) {
+ return allEndpoints.stream().filter(endpoint -> projectDTO.getEndpoints().stream()
+ .anyMatch(endpoint1 -> endpoint1.getName().equals(endpoint.getName())))
+ .collect(Collectors.toList());
+ }
+
+ private Map<String, Map<String, String>> getSharedInfo(String name) {
+ return projectService.get(name).getEndpoints().stream()
+ .collect(Collectors.toMap(ProjectEndpointDTO::getName,
+ endpointDTO -> getSharedInfo(endpointDTO.getEdgeInfo())));
+ }
+
private Map<String, String> getSharedInfo(EdgeInfo edgeInfo) {
Map<String, String> shared = new HashMap<>();
if (Objects.isNull(edgeInfo)) {
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/OdahuServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/OdahuServiceImpl.java
new file mode 100644
index 0000000..3f5671b
--- /dev/null
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/OdahuServiceImpl.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.epam.dlab.backendapi.service.impl;
+
+import com.epam.dlab.auth.UserInfo;
+import com.epam.dlab.backendapi.annotation.BudgetLimited;
+import com.epam.dlab.backendapi.annotation.Project;
+import com.epam.dlab.backendapi.dao.OdahuDAO;
+import com.epam.dlab.backendapi.domain.EndpointDTO;
+import com.epam.dlab.backendapi.domain.OdahuCreateDTO;
+import com.epam.dlab.backendapi.domain.OdahuDTO;
+import com.epam.dlab.backendapi.domain.OdahuFieldsDTO;
+import com.epam.dlab.backendapi.domain.ProjectDTO;
+import com.epam.dlab.backendapi.domain.RequestId;
+import com.epam.dlab.backendapi.service.EndpointService;
+import com.epam.dlab.backendapi.service.OdahuService;
+import com.epam.dlab.backendapi.service.ProjectService;
+import com.epam.dlab.backendapi.util.RequestBuilder;
+import com.epam.dlab.constants.ServiceConsts;
+import com.epam.dlab.dto.UserInstanceStatus;
+import com.epam.dlab.dto.base.odahu.OdahuResult;
+import com.epam.dlab.exceptions.DlabException;
+import com.epam.dlab.exceptions.ResourceConflictException;
+import com.epam.dlab.rest.client.RESTService;
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+@Slf4j
+public class OdahuServiceImpl implements OdahuService {
+
+ private static final String CREATE_ODAHU_API = "infrastructure/odahu";
+ private static final String START_ODAHU_API = "infrastructure/odahu/start";
+ private static final String STOP_ODAHU_API = "infrastructure/odahu/stop";
+ private static final String TERMINATE_ODAHU_API = "infrastructure/odahu/terminate";
+
+ private final ProjectService projectService;
+ private final EndpointService endpointService;
+ private final OdahuDAO odahuDAO;
+ private final RESTService provisioningService;
+ private final RequestBuilder requestBuilder;
+ private final RequestId requestId;
+
+ @Inject
+ public OdahuServiceImpl(ProjectService projectService, EndpointService endpointService, OdahuDAO odahuDAO,
+ @Named(ServiceConsts.PROVISIONING_SERVICE_NAME) RESTService provisioningService,
+ RequestBuilder requestBuilder, RequestId requestId) {
+ this.projectService = projectService;
+ this.endpointService = endpointService;
+ this.odahuDAO = odahuDAO;
+ this.provisioningService = provisioningService;
+ this.requestBuilder = requestBuilder;
+ this.requestId = requestId;
+ }
+
+
+ @Override
+ public List<OdahuDTO> findOdahu() {
+ return odahuDAO.findOdahuClusters();
+ }
+
+ @Override
+ public Optional<OdahuDTO> get(String project, String endpoint) {
+ return odahuDAO.getByProjectEndpoint(project, endpoint);
+ }
+
+ @BudgetLimited
+ @Override
+ public void create(@Project String project, OdahuCreateDTO odahuCreateDTO, UserInfo user) {
+ Optional<OdahuDTO> odahuDTO = odahuDAO.getByProjectEndpoint(odahuCreateDTO.getProject(), odahuCreateDTO.getEndpoint());
+ if (odahuDTO.isPresent()) {
+ throw new ResourceConflictException(String.format("Odahu cluster already exist in system for project %s " +
+ "and endpoint %s", odahuCreateDTO.getProject(), odahuCreateDTO.getEndpoint()));
+ }
+ ProjectDTO projectDTO = projectService.get(project);
+ boolean isAdded = odahuDAO.create(new OdahuDTO(odahuCreateDTO.getName(), odahuCreateDTO.getProject(),
+ odahuCreateDTO.getEndpoint(), UserInstanceStatus.CREATING, getTags(odahuCreateDTO)));
+
+ if (isAdded) {
+ String url = null;
+ EndpointDTO endpointDTO = endpointService.get(odahuCreateDTO.getEndpoint());
+ try {
+ url = endpointDTO.getUrl() + CREATE_ODAHU_API;
+ String uuid =
+ provisioningService.post(url, user.getAccessToken(),
+ requestBuilder.newOdahuCreate(user, odahuCreateDTO, projectDTO, endpointDTO), String.class);
+ requestId.put(user.getName(), uuid);
+ } catch (Exception e) {
+ log.error("Can not perform {} due to: {}, {}", url, e.getMessage(), e);
+ odahuDAO.updateStatus(odahuCreateDTO.getName(), odahuCreateDTO.getProject(), odahuCreateDTO.getEndpoint(),
+ UserInstanceStatus.FAILED);
+ }
+ }
+ }
+
+ @BudgetLimited
+ @Override
+ public void start(String name, @Project String project, String endpoint, UserInfo user) {
+ odahuDAO.updateStatus(name, project, endpoint, UserInstanceStatus.STARTING);
+ actionOnCloud(user, START_ODAHU_API, name, project, endpoint);
+ }
+
+ @Override
+ public void stop(String name, String project, String endpoint, UserInfo user) {
+ odahuDAO.updateStatus(name, project, endpoint, UserInstanceStatus.STOPPING);
+ actionOnCloud(user, STOP_ODAHU_API, name, project, endpoint);
+ }
+
+ @Override
+ public void terminate(String name, String project, String endpoint, UserInfo user) {
+ Optional<OdahuDTO> odahuDTO = get(project, endpoint);
+ if (odahuDTO.isPresent() && UserInstanceStatus.RUNNING == odahuDTO.get().getStatus()) {
+ odahuDAO.updateStatus(name, project, endpoint, UserInstanceStatus.TERMINATING);
+ actionOnCloud(user, TERMINATE_ODAHU_API, name, project, endpoint);
+ } else {
+ log.error("Cannot terminate odahu cluster {}", odahuDTO);
+ throw new DlabException(String.format("Cannot terminate odahu cluster %s", odahuDTO));
+ }
+ }
+
+ @Override
+ public void updateStatus(OdahuResult result, UserInstanceStatus status) {
+ if (Objects.nonNull(result.getResourceUrls()) && !result.getResourceUrls().isEmpty()) {
+ odahuDAO.updateStatusAndUrls(result, status);
+ } else {
+ odahuDAO.updateStatus(result.getName(), result.getProjectName(), result.getEndpointName(), status);
+ }
+ }
+
+ @Override
+ public boolean inProgress(String project, String endpoint) {
+ return get(project, endpoint)
+ .filter(odahu -> Arrays.asList(UserInstanceStatus.CREATING, UserInstanceStatus.STARTING,
+ UserInstanceStatus.STOPPING, UserInstanceStatus.TERMINATING).contains(odahu.getStatus()))
+ .isPresent();
+ }
+
+ private void actionOnCloud(UserInfo user, String uri, String name, String project, String endpoint) {
+ String url = null;
+ EndpointDTO endpointDTO = endpointService.get(endpoint);
+ ProjectDTO projectDTO = projectService.get(project);
+ try {
+ OdahuFieldsDTO fields = odahuDAO.getFields(name, project, endpoint);
+ url = endpointDTO.getUrl() + uri;
+ String uuid =
+ provisioningService.post(url, user.getAccessToken(),
+ requestBuilder.newOdahuAction(user, name, projectDTO, endpointDTO, fields), String.class);
+ requestId.put(user.getName(), uuid);
+ } catch (Exception e) {
+ log.error("Can not perform {} due to: {}, {}", url, e.getMessage(), e);
+ odahuDAO.updateStatus(name, project, project, UserInstanceStatus.FAILED);
+ }
+ }
+
+ private Map<String, String> getTags(OdahuCreateDTO odahuCreateDTO) {
+ Map<String, String> tags = new HashMap<>();
+ tags.put("custom_tag", odahuCreateDTO.getCustomTag());
+ tags.put("project_tag", odahuCreateDTO.getProject());
+ tags.put("endpoint_tag", odahuCreateDTO.getEndpoint());
+ return tags;
+ }
+}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ProjectServiceImpl.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ProjectServiceImpl.java
index 0e5a2d5..ca4effa 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ProjectServiceImpl.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/service/impl/ProjectServiceImpl.java
@@ -14,8 +14,8 @@
import com.epam.dlab.backendapi.domain.UpdateProjectDTO;
import com.epam.dlab.backendapi.service.EndpointService;
import com.epam.dlab.backendapi.service.ExploratoryService;
+import com.epam.dlab.backendapi.service.OdahuService;
import com.epam.dlab.backendapi.service.ProjectService;
-import com.epam.dlab.backendapi.service.SecurityService;
import com.epam.dlab.backendapi.util.RequestBuilder;
import com.epam.dlab.constants.ServiceConsts;
import com.epam.dlab.dto.UserInstanceStatus;
@@ -55,14 +55,14 @@
private final RequestBuilder requestBuilder;
private final EndpointService endpointService;
private final ExploratoryDAO exploratoryDAO;
- private final SecurityService securityService;
+ private final OdahuService odahuService;
@Inject
public ProjectServiceImpl(ProjectDAO projectDAO, ExploratoryService exploratoryService,
UserGroupDao userGroupDao,
@Named(ServiceConsts.PROVISIONING_SERVICE_NAME) RESTService provisioningService,
RequestId requestId, RequestBuilder requestBuilder, EndpointService endpointService,
- ExploratoryDAO exploratoryDAO, SecurityService securityService) {
+ ExploratoryDAO exploratoryDAO, OdahuService odahuService) {
this.projectDAO = projectDAO;
this.exploratoryService = exploratoryService;
this.userGroupDao = userGroupDao;
@@ -71,7 +71,7 @@
this.requestBuilder = requestBuilder;
this.endpointService = endpointService;
this.exploratoryDAO = exploratoryDAO;
- this.securityService = securityService;
+ this.odahuService = odahuService;
}
@Override
@@ -124,10 +124,18 @@
projectActionOnCloud(userInfo, name, TERMINATE_PRJ_API, endpoint);
projectDAO.updateEdgeStatus(name, endpoint, UserInstanceStatus.TERMINATING);
exploratoryService.updateProjectExploratoryStatuses(name, endpoint, UserInstanceStatus.TERMINATING);
+ odahuService.get(name, endpoint)
+ .filter(o -> UserInstanceStatus.RUNNING == o.getStatus())
+ .ifPresent(odahu -> odahuService.terminate(odahu.getName(), name, endpoint, userInfo));
}
@Override
public void terminateEndpoint(UserInfo userInfo, List<String> endpoints, String name) {
+ List<ProjectEndpointDTO> projectEndpoint = get(name).getEndpoints().stream()
+ .filter(e -> endpoints.contains(e.getName()))
+ .collect(Collectors.toList());
+ checkProjectRelatedResourcesInProgress(name, projectEndpoint, TERMINATE_ACTION);
+
endpoints.forEach(endpoint -> terminateEndpoint(userInfo, endpoint, name));
}
@@ -262,12 +270,13 @@
}
private void checkProjectRelatedResourcesInProgress(String projectName, List<ProjectEndpointDTO> endpoints, String action) {
- boolean edgeProgress = endpoints.stream().anyMatch(e ->
- Arrays.asList(UserInstanceStatus.CREATING, UserInstanceStatus.STARTING, UserInstanceStatus.STOPPING,
- UserInstanceStatus.TERMINATING).contains(e.getStatus()));
+ boolean edgeAndOdahuProgress = endpoints.stream().anyMatch(e ->
+ Arrays.asList(UserInstanceStatus.CREATING, UserInstanceStatus.STARTING, UserInstanceStatus.STOPPING,
+ UserInstanceStatus.TERMINATING).contains(e.getStatus())
+ || odahuService.inProgress(projectName, e.getName()));
List<String> endpointsName = endpoints.stream().map(ProjectEndpointDTO::getName).collect(Collectors.toList());
- if (edgeProgress || !checkExploratoriesAndComputationalProgress(projectName, endpointsName)) {
+ if (edgeAndOdahuProgress || !checkExploratoriesAndComputationalProgress(projectName, endpointsName)) {
throw new ResourceConflictException((String.format("Can not %s environment because one of project " +
"resource is in processing stage", action)));
}
diff --git a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/RequestBuilder.java b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/RequestBuilder.java
index afe06cd..8e9f3d8 100644
--- a/services/self-service/src/main/java/com/epam/dlab/backendapi/util/RequestBuilder.java
+++ b/services/self-service/src/main/java/com/epam/dlab/backendapi/util/RequestBuilder.java
@@ -24,6 +24,8 @@
import com.epam.dlab.backendapi.dao.SettingsDAO;
import com.epam.dlab.backendapi.domain.EndpointDTO;
import com.epam.dlab.backendapi.domain.ExploratoryLibCache;
+import com.epam.dlab.backendapi.domain.OdahuCreateDTO;
+import com.epam.dlab.backendapi.domain.OdahuFieldsDTO;
import com.epam.dlab.backendapi.domain.ProjectDTO;
import com.epam.dlab.backendapi.resources.dto.BackupFormDTO;
import com.epam.dlab.backendapi.resources.dto.ComputationalCreateFormDTO;
@@ -70,6 +72,8 @@
import com.epam.dlab.dto.gcp.computational.GcpComputationalTerminateDTO;
import com.epam.dlab.dto.gcp.computational.SparkComputationalCreateGcp;
import com.epam.dlab.dto.gcp.exploratory.ExploratoryCreateGcp;
+import com.epam.dlab.dto.odahu.ActionOdahuDTO;
+import com.epam.dlab.dto.odahu.CreateOdahuDTO;
import com.epam.dlab.dto.project.ProjectActionDTO;
import com.epam.dlab.dto.project.ProjectCreateDTO;
import com.epam.dlab.exceptions.DlabException;
@@ -626,6 +630,33 @@
.withCloudSettings(cloudSettings(userInfo, endpointDTO.getCloudProvider()));
}
+ public CreateOdahuDTO newOdahuCreate(UserInfo user, OdahuCreateDTO odahuCreateDTO, ProjectDTO projectDTO, EndpointDTO endpointDTO) {
+ return CreateOdahuDTO.builder()
+ .name(odahuCreateDTO.getName())
+ .project(projectDTO.getName())
+ .endpoint(odahuCreateDTO.getEndpoint())
+ .key(projectDTO.getKey().replace("\n", ""))
+ .build()
+ .withEdgeUserName(getEdgeUserName(user, endpointDTO.getCloudProvider()))
+ .withCloudSettings(cloudSettings(user, endpointDTO.getCloudProvider()));
+ }
+
+ public ActionOdahuDTO newOdahuAction(UserInfo user, String name, ProjectDTO projectDTO, EndpointDTO endpointDTO,
+ OdahuFieldsDTO odahuFields) {
+ return ActionOdahuDTO.builder()
+ .name(name)
+ .project(projectDTO.getName())
+ .key(projectDTO.getKey().replace("\n", ""))
+ .endpoint(endpointDTO.getName())
+ .grafanaAdmin(odahuFields.getGrafanaAdmin())
+ .grafanaPassword(odahuFields.getGrafanaPassword())
+ .oauthCookieSecret(odahuFields.getOauthCookieSecret())
+ .decryptToken(odahuFields.getDecryptToken())
+ .build()
+ .withEdgeUserName(getEdgeUserName(user, endpointDTO.getCloudProvider()))
+ .withCloudSettings(cloudSettings(user, endpointDTO.getCloudProvider()));
+ }
+
/**
* Returns application name basing on docker image
*
diff --git a/services/self-service/src/main/resources/mongo/aws/mongo_roles.json b/services/self-service/src/main/resources/mongo/aws/mongo_roles.json
index 9998d84..2adb28c 100644
--- a/services/self-service/src/main/resources/mongo/aws/mongo_roles.json
+++ b/services/self-service/src/main/resources/mongo/aws/mongo_roles.json
@@ -335,7 +335,8 @@
"/api/settings",
"/user/settings",
"/api/project",
- "/api/endpoint"
+ "/api/endpoint",
+ "/api/odahu"
],
"groups": [
"$anyuser"
diff --git a/services/self-service/src/main/resources/mongo/azure/mongo_roles.json b/services/self-service/src/main/resources/mongo/azure/mongo_roles.json
index 113a705..bbd7789 100644
--- a/services/self-service/src/main/resources/mongo/azure/mongo_roles.json
+++ b/services/self-service/src/main/resources/mongo/azure/mongo_roles.json
@@ -275,7 +275,8 @@
"/api/settings",
"/user/settings",
"/api/project",
- "/api/endpoint"
+ "/api/endpoint",
+ "/api/odahu"
],
"groups": [
"$anyuser"
diff --git a/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json b/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json
index 8098628..d55e85f 100644
--- a/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json
+++ b/services/self-service/src/main/resources/mongo/gcp/mongo_roles.json
@@ -311,7 +311,8 @@
"/api/settings",
"/user/settings",
"/api/project",
- "/api/endpoint"
+ "/api/endpoint",
+ "/api/odahu"
],
"groups": [
"$anyuser"
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/administration.module.ts b/services/self-service/src/main/resources/webapp/src/app/administration/administration.module.ts
index e535bda..8c19a94 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/administration.module.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/administration.module.ts
@@ -23,10 +23,10 @@
import { ManagenementModule } from './management';
import { ProjectModule } from './project';
import { RolesModule } from './roles';
+import {LegionDeploymentModule} from "./legion-deployment";
@NgModule({
- imports: [CommonModule, ManagenementModule, ProjectModule, RolesModule],
- declarations: [],
- exports: [ManagenementModule, ProjectModule, RolesModule]
+ imports: [CommonModule, ManagenementModule, ProjectModule, RolesModule, LegionDeploymentModule],
+ exports: [ManagenementModule, ProjectModule, RolesModule, LegionDeploymentModule],
})
export class AdministrationModule { }
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.html b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.html
new file mode 100644
index 0000000..ff04e55
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.html
@@ -0,0 +1,93 @@
+<div class="create-legion-cluster" id="dialog-box">
+ <header class="dialog-header">
+ <h4 class="modal-title">Create Odahu cluster</h4>
+ <button type="button" class="close" (click)="dialogRef.close()">×</button>
+ </header>
+ <div class="dialog-content selection">
+ <div id="scrolling" class="content-box mat-reset scrolling-content">
+ <form [formGroup]="createLegionClusterForm" *ngIf="createLegionClusterForm" novalidate>
+ <div class="control-group">
+ <label class="label">Select project</label>
+ <div class="control selector-wrapper">
+ <mat-form-field>
+ <mat-label>Select project</mat-label>
+ <mat-select formControlName="project" panelClass="create-resources-dialog">
+ <mat-option *ngFor="let project of projects" [value]="project.name" (click)="setEndpoints(project)">
+ {{ project.name }}</mat-option>
+ <mat-option *ngIf="!projects.length" class="multiple-select ml-10" disabled>
+ No projects for creating Odahu clusters
+ </mat-option>
+ </mat-select>
+ <button class="caret">
+ <i class="material-icons">keyboard_arrow_down</i>
+ </button>
+ </mat-form-field>
+ </div>
+ </div>
+
+ <div class="control-group">
+ <label class="label">Select endpoint</label>
+ <div class="control selector-wrapper" [ngClass]="{ 'not-active' : !endpoints.length }">
+ <mat-form-field>
+ <mat-label>Select endpoint</mat-label>
+ <mat-select formControlName="endpoint" disableOptionCentering [disabled]="!endpoints.length"
+ panelClass="create-resources-dialog">
+ <mat-option *ngFor="let endpoint of endpoints" [value]="endpoint">
+ {{ endpoint }}
+ </mat-option>
+ <mat-option *ngIf="!endpoints.length" class="multiple-select ml-10" disabled>Endpoints list is empty</mat-option>
+ </mat-select>
+ <button class="caret">
+ <i class="material-icons">keyboard_arrow_down</i>
+ </button>
+ </mat-form-field>
+ </div>
+ </div>
+
+ <div class="control-group name-control">
+ <label class="label">Name</label>
+ <div class="control">
+ <input type="text" class="form-control" placeholder="Enter Name" formControlName="name">
+ <span class="error" *ngIf="!createLegionClusterForm.controls.name.valid && createLegionClusterForm.controls.name.dirty && !createLegionClusterForm.controls.name.hasError('duplication')">
+ Odahu cluster name can only contain letters and numbers
+ </span>
+ <span class="error" *ngIf="createLegionClusterForm.controls.name.hasError('duplication')">This Odahu cluster name already exists.</span>
+ </div>
+ </div>
+
+ <div class="control-group name-control">
+ <label class="label">Custom tag</label>
+ <div class="control">
+ <input type="text" class="form-control" placeholder="Enter custom tag" formControlName="custom_tag">
+ <span class="error"
+ *ngIf="!createLegionClusterForm.controls.custom_tag.valid && createLegionClusterForm.controls.custom_tag.dirty">
+ Custom tag can only contain letters, numbers, hyphens and '_' but can not end with special characters</span>
+ </div>
+ </div>
+
+<!-- <div class="control-group">-->
+<!-- <label class="label" [ngStyle]="!createLegionClusterForm.controls.useExistingClusterUrl.value && {'width': '100%' }">-->
+<!-- <input type="checkbox" formControlName="useExistingClusterUrl"/> Use existing k8s cluster-->
+<!-- </label>-->
+<!-- <div class="control" *ngIf="createLegionClusterForm.controls.useExistingClusterUrl.value">-->
+<!-- <input type="text" class="form-control"-->
+<!-- formControlName="existingClusterUrl" placeholder="Enter ODAHU k8s cluster URL"/>-->
+<!-- <span class="error url-error">-->
+<!-- <span *ngIf="!createLegionClusterForm.controls.existingClusterUrl.valid">Please enter valid cluster URL</span>-->
+<!-- </span>-->
+<!-- </div>-->
+<!-- </div>-->
+
+ <div class="text-center m-top-30">
+ <button mat-raised-button type="button" class="butt action" (click)="dialogRef.close()">Cancel</button>
+ <button mat-raised-button type="button" class="butt butt-success action"
+ [disabled]="!createLegionClusterForm.valid" (click)="createOdahuCluster(createLegionClusterForm.value)">
+ Create
+ </button>
+ </div>
+
+ </form>
+ </div>
+ </div>
+</div>
+
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.scss b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.scss
new file mode 100644
index 0000000..324f0df
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.scss
@@ -0,0 +1,7 @@
+.create-legion-cluster{
+ .error{
+ position: absolute;
+ left: 0;
+ top: 36px;
+ }
+}
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.ts
new file mode 100644
index 0000000..7aa941d
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/create-legion-cluster.component.ts
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, OnInit, Inject } from '@angular/core';
+import { FormGroup, FormBuilder, Validators } from '@angular/forms';
+import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
+import { ToastrService } from 'ngx-toastr';
+
+import { Project } from '../../project/project.component';
+import { ProjectService, LegionDeploymentService } from '../../../core/services';
+
+import { DICTIONARY } from '../../../../dictionary/global.dictionary';
+import {CheckUtils, PATTERNS} from '../../../core/util';
+
+
+@Component({
+ selector: 'create-legion-cluster',
+ templateUrl: 'create-legion-cluster.component.html',
+ styleUrls: ['./create-legion-cluster.component.scss']
+})
+
+export class CreateLegionClusterComponent implements OnInit {
+ readonly DICTIONARY = DICTIONARY;
+ public createLegionClusterForm: FormGroup;
+
+ projects: Project[] = [];
+ endpoints: Array<String> = [];
+
+ constructor(
+ @Inject(MAT_DIALOG_DATA) public data: any,
+ public toastr: ToastrService,
+ public dialogRef: MatDialogRef<CreateLegionClusterComponent>,
+ private _fb: FormBuilder,
+ private projectService: ProjectService,
+ private legionDeploymentService: LegionDeploymentService,
+ ) {
+ }
+
+ ngOnInit() {
+ this.getUserProjects();
+ this.initFormModel();
+ }
+
+ public getUserProjects(): void {
+ this.projectService.getUserProjectsList(true).subscribe((projects: any) => {
+ this.projects = projects.filter(project => {
+ return project.endpoints.length > project.odahu.filter(od => od.status !== 'FAILED' && od.status !== 'TERMINATED').length; }
+ );
+ });
+ }
+
+ public setEndpoints(project): void {
+ this.endpoints = project.endpoints
+ .filter(e => e.status === 'RUNNING' && !this.data.some(odahu => odahu.status !== 'FAILED'
+ && odahu.status !== 'TERMINATED'
+ && odahu.endpoint === e.name
+ && odahu.project === project.name)
+ )
+ .map(e => e.name);
+ }
+
+ private initFormModel(): void {
+ this.createLegionClusterForm = this._fb.group({
+ name: ['', [Validators.required, Validators.pattern(PATTERNS.namePattern), this.checkDuplication.bind(this)]],
+ project: ['', Validators.required],
+ endpoint: ['', [Validators.required]],
+ custom_tag: ['', [Validators.pattern(PATTERNS.namePattern)]]
+ });
+ }
+
+ private createOdahuCluster(value): void {
+ this.dialogRef.close();
+ this.legionDeploymentService.createOdahuNewCluster(value).subscribe(() => {
+ this.toastr.success('Odahu cluster creation is processing!', 'Success!');
+ }, error => this.toastr.error(error.message || 'Odahu cluster creation failed!', 'Oops!')
+ );
+ }
+
+ private checkDuplication(control) {
+ if (control && control.value) {
+ for (let index = 0; index < this.data.length; index++) {
+ if (CheckUtils.delimitersFiltering(control.value) === CheckUtils.delimitersFiltering(this.data[index].name))
+ return { duplication: true };
+ }
+ }
+ }
+}
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/index.ts b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/index.ts
new file mode 100644
index 0000000..a3adc9c
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/create-legion-claster/index.ts
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { NgModule } from '@angular/core';
+import { CommonModule } from '@angular/common';
+import { FormsModule, ReactiveFormsModule } from '@angular/forms';
+
+import { MaterialModule } from '../../../shared/material.module';
+import { FormControlsModule } from '../../../shared/form-controls';
+import { KeysPipeModule, UnderscorelessPipeModule } from '../../../core/pipes';
+import {CreateLegionClusterComponent} from "./create-legion-cluster.component";
+
+export * from './create-legion-cluster.component';
+
+@NgModule({
+ imports: [
+ CommonModule,
+ FormsModule,
+ ReactiveFormsModule,
+ FormControlsModule,
+ MaterialModule,
+ KeysPipeModule,
+ UnderscorelessPipeModule
+ ],
+ declarations: [CreateLegionClusterComponent],
+ entryComponents: [CreateLegionClusterComponent],
+ exports: [CreateLegionClusterComponent]
+})
+export class CreateLegionClusterModule { }
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/index.ts b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/index.ts
new file mode 100644
index 0000000..fa28e36
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/index.ts
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { NgModule } from '@angular/core';
+import { CommonModule } from '@angular/common';
+import { FormsModule, ReactiveFormsModule } from '@angular/forms';
+
+import { MaterialModule } from '../../shared/material.module';
+import { FormControlsModule } from '../../shared/form-controls';
+import { UnderscorelessPipeModule } from '../../core/pipes/underscoreless-pipe';
+
+import {BubbleModule} from "../../shared/bubble";
+import {LegionDeploymentComponent} from "./legion-deployment.component";
+import {LegionDeploymentDataService} from "./legion-deployment-data.service";
+import {LegionListComponent} from "./legion-list/legion-list.component";
+import {CreateLegionClusterModule} from "./create-legion-claster";
+
+@NgModule({
+ imports: [
+ CommonModule,
+ FormsModule,
+ ReactiveFormsModule,
+ MaterialModule,
+ FormControlsModule,
+ UnderscorelessPipeModule,
+ BubbleModule,
+ CreateLegionClusterModule
+ ],
+ declarations: [LegionDeploymentComponent, LegionListComponent],
+ entryComponents: [],
+ providers: [LegionDeploymentDataService],
+ exports: [LegionDeploymentComponent]
+})
+export class LegionDeploymentModule { }
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment-data.service.ts b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment-data.service.ts
new file mode 100644
index 0000000..83dfeaf
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment-data.service.ts
@@ -0,0 +1,27 @@
+import { Injectable } from '@angular/core';
+import {BehaviorSubject, Observable} from 'rxjs';
+import {LegionDeploymentService} from '../../core/services';
+
+
+@Injectable({
+ providedIn: 'root'
+})
+
+export class LegionDeploymentDataService {
+ _legionClasters = new BehaviorSubject<any>(null);
+
+ constructor(private legionDeploymentService: LegionDeploymentService) {
+ this.getClastersList();
+ }
+
+ public updateClasters(): void {
+ this.getClastersList();
+ }
+
+ private getClastersList(): void {
+ this.legionDeploymentService.getOduhuClustersList().subscribe(
+ (response: any ) => {
+ return this._legionClasters.next(response);
+ });
+ }
+}
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.html b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.html
new file mode 100644
index 0000000..1368180
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.html
@@ -0,0 +1,42 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+
+<div class="base-retreat">
+ <div class="sub-nav">
+ <div>
+ <button mat-raised-button class="butt butt-create" (click)="createLegionCluster()">
+ <i class="material-icons">add</i>Create new
+ </button>
+ </div>
+ <div>
+ <button mat-raised-button class="butt" (click)="refreshGrid()">
+ <i class="material-icons">autorenew</i>Refresh
+ </button>
+ </div>
+ </div>
+
+ <mat-divider></mat-divider>
+
+ <div>
+ <legion-list>
+ </legion-list>
+ </div>
+</div>
+
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.scss b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.scss
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.scss
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.ts
new file mode 100644
index 0000000..263debf
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-deployment.component.ts
@@ -0,0 +1,59 @@
+import { Component, OnInit } from '@angular/core';
+import {LegionDeploymentDataService} from './legion-deployment-data.service';
+import {Subscription} from 'rxjs';
+import {MatDialog} from '@angular/material/dialog';
+import {ToastrService} from 'ngx-toastr';
+import {CreateLegionClusterComponent} from './create-legion-claster/create-legion-cluster.component';
+import {HealthStatusService, LegionDeploymentService} from '../../core/services';
+
+export interface OdahuCluster {
+ name: string;
+ project: string;
+ endpoint: string;
+}
+
+@Component({
+ selector: 'dlab-legion-deployment',
+ templateUrl: './legion-deployment.component.html',
+ styleUrls: ['./legion-deployment.component.scss']
+})
+export class LegionDeploymentComponent implements OnInit {
+
+ private legionClastersList: any[];
+ private subscriptions: Subscription = new Subscription();
+ private healthStatus;
+
+ constructor(
+ private legionDeploymentDataService: LegionDeploymentDataService,
+ private dialog: MatDialog,
+ public toastr: ToastrService,
+ public legionDeploymentService: LegionDeploymentService,
+ private healthStatusService: HealthStatusService,
+ ) { }
+
+ ngOnInit() {
+ this.getEnvironmentHealthStatus();
+ this.subscriptions.add(this.legionDeploymentDataService._legionClasters.subscribe(
+ (value) => {
+ if (value) this.legionClastersList = value;
+ }));
+ this.refreshGrid();
+ }
+
+ public createLegionCluster(): void {
+ this.dialog.open(CreateLegionClusterComponent, { data: this.legionClastersList, panelClass: 'modal-lg' })
+ .afterClosed().subscribe((result) => {
+ result && this.getEnvironmentHealthStatus();
+ this.refreshGrid();
+ });
+ }
+
+ private getEnvironmentHealthStatus(): void {
+ this.healthStatusService.getEnvironmentHealthStatus()
+ .subscribe((result: any) => this.healthStatus = result);
+ }
+
+ public refreshGrid(): void {
+ this.legionDeploymentDataService.updateClasters();
+ }
+}
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.html b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.html
new file mode 100644
index 0000000..a670b2f
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.html
@@ -0,0 +1,94 @@
+
+<table mat-table [dataSource]="dataSource" class="legion-clasters-table mat-elevation-z6 selection">
+ <ng-container matColumnDef="project">
+ <th mat-header-cell *matHeaderCellDef class="project"> Project name </th>
+ <td mat-cell *matCellDef="let element" class="project">
+ <span *ngIf="element && element.project">{{element.project}}</span>
+ </td>
+ <td mat-footer-cell *matFooterCellDef></td>
+ </ng-container>
+
+ <ng-container matColumnDef="endpoint-url">
+ <th mat-header-cell *matHeaderCellDef class="endpoint"> Endpoint </th>
+ <td mat-cell *matCellDef="let element" class="endpoint">
+ <span *ngIf="element && element.endpoint" matTooltip="{{element.endpoint}}" [matTooltipPosition]="'above'">
+ {{element.endpoint}}
+ </span>
+ </td>
+ <td mat-footer-cell *matFooterCellDef></td>
+ </ng-container>
+
+ <ng-container matColumnDef="legion-name">
+ <th mat-header-cell *matHeaderCellDef class="legion-name"> Odahu cluster name </th>
+ <td mat-cell *matCellDef="let element" class="legion-name">
+ <span *ngIf="element && element.name">{{element.name}}</span>
+ </td>
+ <td mat-footer-cell *matFooterCellDef></td>
+ </ng-container>
+
+ <ng-container matColumnDef="legion-status">
+ <th mat-header-cell *matHeaderCellDef class="legion-status"> Odahu cluster status </th>
+ <td mat-cell *matCellDef="let element" class="legion-status">
+ <span *ngIf="element && element.name" [ngClass]="element.status.toLowerCase()">{{ element.status | titlecase}}</span>
+ </td>
+ <td mat-footer-cell *matFooterCellDef></td>
+ </ng-container>
+
+ <ng-container matColumnDef="actions">
+ <th mat-header-cell *matHeaderCellDef class="legion-actions"></th>
+ <td mat-cell *matCellDef="let element" class="settings">
+ <span *ngIf="element && element.name" #settings (click)="actions.toggle($event, settings)" class="actions" [ngClass]="{'disabled': element.status !== 'RUNNING' && element.status !== 'STOPPED'}"></span>
+ <bubble-up #actions class="list-menu" position="bottom-left" alternative="top-left">
+ <ul class="list-unstyled">
+ <div class="active-items"></div>
+ <li class="project-seting-item" *ngIf="element.status === 'STOPPED'" (click)="odahuAction(element, 'start')">
+ <i class="material-icons">play_circle_outline</i>
+ <a class="action">
+ Start
+ </a>
+ </li>
+ <li class="project-seting-item" *ngIf="element.status === 'RUNNING'" (click)="odahuAction(element, 'stop')">
+ <i class="material-icons">pause_circle_outline</i>
+ <a class="action" >
+ Stop
+ </a>
+ </li>
+ <li class="project-seting-item" [ngClass]="{'disabled' : element.status === 'STOPPED'}" *ngIf="element.status !== 'TERMINATED' && element.status !== 'TERMINATING'" (click)="odahuAction(element, 'terminate')">
+ <i class="material-icons">phonelink_off</i>
+ <a class="action">
+ Terminate
+ </a>
+ </li>
+ <!--<li class="project-seting-item">-->
+ <!--<i class="material-icons">arrow_downward</i>-->
+ <!--<a>-->
+ <!--Scale-down-->
+ <!--</a>-->
+ <!--</li>-->
+ <!--<li class="project-seting-item">-->
+ <!--<i class="material-icons">arrow_upward</i>-->
+ <!--<a class="action">-->
+ <!--Scale-up-->
+ <!--</a>-->
+ <!--</li>-->
+ </ul>
+ </bubble-up>
+ </td>
+ <td mat-footer-cell *matFooterCellDef></td>
+ </ng-container>
+
+ <ng-container matColumnDef="placeholder">
+ <td mat-footer-cell *matFooterCellDef
+ [colSpan]="displayedColumns.length"
+ class="info">
+ <span>No Odahu clusters</span>
+ </td>
+ <td mat-footer-cell *matFooterCellDef></td>
+ </ng-container>
+
+ <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
+ <tr mat-row *matRowDef="let row; columns: displayedColumns;"></tr>
+ <tr [hidden]="legionClustersList && legionClustersList.length" mat-footer-row *matFooterRowDef="['placeholder']"></tr>
+</table>
+
+
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.scss b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.scss
new file mode 100644
index 0000000..21fc43e
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.scss
@@ -0,0 +1,77 @@
+.legion-clasters-table {
+ width: 100%;
+
+ .project, .endpoint, .legion-name, .legion-status{
+ width: 23%;
+ }
+
+ td.settings {
+ position: relative;
+ vertical-align: middle !important;
+ text-align: right;
+ }
+
+ .list-menu {
+ min-width: 190px;
+ }
+
+ .material-icons {
+ font-size: 18px;
+ padding-top: 1px;
+ }
+
+ td.settings .actions {
+ background-image: url(../../../../assets/svg/settings_icon.svg);
+ width: 16px;
+ height: 16px;
+ display: inline-block;
+ text-align: center;
+ cursor: pointer;
+ &.disabled {
+ opacity: 0.4;
+ cursor: not-allowed;
+ pointer-events: none;
+ }
+ }
+
+ td {
+ &.info.mat-footer-cell{
+ text-align: center;
+ padding: 40px;
+ }
+ .settings {
+ position: relative;
+ vertical-align: middle !important;
+ text-align: right;
+
+ .actions {
+ background-image: url(../../../../assets/svg/settings_icon.svg);
+ width: 16px;
+ height: 16px;
+ display: inline-block;
+ text-align: center;
+ cursor: pointer;
+ }
+ }
+
+ .project-seting-item {
+ display: flex;
+ padding: 10px;
+ align-items: center;
+ border-bottom: 1px solid #edf1f5;
+ cursor: pointer;
+ color: #577289;
+
+ &:hover {
+ color: #36afd5;
+ transition: all .45s ease-in-out;
+ }
+
+ a {
+ padding-left: 5px;
+ }
+ }
+ }
+}
+
+
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.ts
new file mode 100644
index 0000000..7e7913e
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/legion-deployment/legion-list/legion-list.component.ts
@@ -0,0 +1,48 @@
+import { Component, OnInit } from '@angular/core';
+import {Subscription} from 'rxjs';
+import {LegionDeploymentDataService} from '../legion-deployment-data.service';
+import { MatTableDataSource } from '@angular/material/table';
+import {LegionDeploymentService} from '../../../core/services';
+import {ToastrService} from 'ngx-toastr';
+import {MatDialog} from '@angular/material/dialog';
+import {OdahuActionDialogComponent} from '../../../shared/modal-dialog/odahu-action-dialog';
+
+@Component({
+ selector: 'legion-list',
+ templateUrl: './legion-list.component.html',
+ styleUrls: ['./legion-list.component.scss']
+})
+export class LegionListComponent implements OnInit {
+ private legionClustersList: any[];
+ private subscriptions: Subscription = new Subscription();
+ public dataSource: MatTableDataSource<any>;
+ displayedColumns: string[] = [ 'legion-name', 'project', 'endpoint-url', 'legion-status', 'actions'];
+
+ constructor(
+ private legionDeploymentDataService: LegionDeploymentDataService,
+ private legionDeploymentService: LegionDeploymentService,
+ public toastr: ToastrService,
+ public dialog: MatDialog
+ ) { }
+
+ ngOnInit() {
+ this.subscriptions.add(this.legionDeploymentDataService._legionClasters.subscribe(
+ (value) => {
+ if (value) {
+ this.legionClustersList = value;
+ this.dataSource = new MatTableDataSource(value);
+ }
+ }));
+ }
+
+ private odahuAction(element: any, action: string) {
+ this.dialog.open(OdahuActionDialogComponent, {data: {type: action, item: element}, panelClass: 'modal-sm'})
+ .afterClosed().subscribe(result => {
+ result && this.legionDeploymentService.odahuAction(element, action).subscribe(v =>
+ this.legionDeploymentDataService.updateClasters(),
+ error => this.toastr.error(`Odahu cluster ${action} failed!`, 'Oops!')
+ ) ;
+ }, error => this.toastr.error(error.message || `Odahu cluster ${action} failed!`, 'Oops!')
+ );
+ }
+}
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.html b/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.html
index 631e7ae..1945cd0 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/management/management-grid/management-grid.component.html
@@ -121,12 +121,12 @@
</td>
</ng-container>
- <ng-container matColumnDef="actions">
+ <ng-container matColumnDef="actions">
<th mat-header-cell *matHeaderCellDef class="actions">
<span class="label"> Actions </span>
</th>
- <td mat-cell *matCellDef="let element" class="settings actions-col">
- <span #settings class="actions" (click)="actions.toggle($event, settings)" *ngIf="element.type !== 'edge node'"
+ <td mat-cell *matCellDef="let element" class=" settings actions-col">
+ <span #settings class="actions" (click)="actions.toggle($event, settings)" *ngIf="element.type !== 'edge node' && element.type !== 'odahu'"
[ngClass]="{
'disabled' : isActiveResources(element),
'disabled' : element.status !== 'running' && element.status !== 'stopped' && element.status !== 'stopping' && element.status !== 'failed' }"></span>
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/project/index.ts b/services/self-service/src/main/resources/webapp/src/app/administration/project/index.ts
index 609ea59..8203e55 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/project/index.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/project/index.ts
@@ -30,7 +30,7 @@
import { ProjectComponent, EditProjectComponent } from './project.component';
import { ProjectDataService } from './project-data.service';
-import {BubbleModule} from "../../shared/bubble";
+import {BubbleModule} from '../../shared/bubble';
@NgModule({
imports: [
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/project/project-form/project-form.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/project/project-form/project-form.component.ts
index 683e7d7..f38ab06 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/project/project-form/project-form.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/project/project-form/project-form.component.ts
@@ -29,7 +29,7 @@
import { Project } from '../project.component';
import { DICTIONARY } from '../../../../dictionary/global.dictionary';
-export interface GenerateKey { privateKey: string, publicKey: string };
+export interface GenerateKey { privateKey: string, publicKey: string }
@Component({
selector: 'project-form',
@@ -139,7 +139,7 @@
}
public selectOptions(list, key, select?) {
- let filter = key === 'endpoints' ? list.map(el => el.name) : list;
+ const filter = key === 'endpoints' ? list.map(el => el.name) : list;
this.projectForm.controls[key].setValue(select ? filter : []);
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/administration/project/project-list/project-list.component.ts b/services/self-service/src/main/resources/webapp/src/app/administration/project/project-list/project-list.component.ts
index 9238b7f..e917f04 100644
--- a/services/self-service/src/main/resources/webapp/src/app/administration/project/project-list/project-list.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/administration/project/project-list/project-list.component.ts
@@ -28,7 +28,7 @@
import { CheckUtils } from '../../../core/util';
import {ProgressBarService} from '../../../core/services/progress-bar.service';
import {EdgeActionDialogComponent} from '../../../shared/modal-dialog/edge-action-dialog';
-import {EndpointService} from '../../../core/services';
+import {EndpointService} from "../../../core/services";
@Component({
@@ -50,6 +50,7 @@
public toastr: ToastrService,
private projectDataService: ProjectDataService,
private progressBarService: ProgressBarService,
+ private endpointService: EndpointService,
@Inject(MAT_DIALOG_DATA) public data: any,
public dialogRef: MatDialogRef<ProjectListComponent>,
public dialog: MatDialog,
@@ -69,6 +70,18 @@
setTimeout(() => {this.progressBarService.startProgressBar(); } , 0);
this.subscriptions.add(this.projectDataService._projects.subscribe((value: Project[]) => {
this.projectList = value;
+ this.endpointService.getEndpointsData().subscribe((endpoints: any) => {
+ if (this.projectList) {
+ this.projectList.forEach(project => project.endpoints.forEach(endpoint => {
+ const filtredEndpoints = endpoints.filter(v => v.name === endpoint.name);
+ if (filtredEndpoints.length) {
+ endpoint.endpointStatus = endpoints.filter(v => v.name === endpoint.name)[0].status;
+ } else {
+ endpoint.endpointStatus = 'N/A';
+ }
+ }));
+ }
+ });
if (value) this.dataSource = new MatTableDataSource(value);
this.progressBarService.stopProgressBar();
}, () => this.progressBarService.stopProgressBar()));
diff --git a/services/self-service/src/main/resources/webapp/src/app/app.routing.module.ts b/services/self-service/src/main/resources/webapp/src/app/app.routing.module.ts
index 3c8ae3f..9afd6a5 100644
--- a/services/self-service/src/main/resources/webapp/src/app/app.routing.module.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/app.routing.module.ts
@@ -34,6 +34,7 @@
import { SwaggerComponent } from './swagger/swagger.component';
import { AuthorizationGuard, CheckParamsGuard, CloudProviderGuard, AdminGuard } from './core/services';
+import {LegionDeploymentComponent} from "./administration/legion-deployment/legion-deployment.component";
const routes: Routes = [{
path: 'login',
@@ -59,6 +60,11 @@
path: 'projects',
component: ProjectComponent,
canActivate: [AuthorizationGuard, AdminGuard],
+ },
+ {
+ path: 'odahu',
+ component: LegionDeploymentComponent,
+ canActivate: [AuthorizationGuard,],
}, {
path: 'roles',
component: RolesComponent,
diff --git a/services/self-service/src/main/resources/webapp/src/app/core/core.module.ts b/services/self-service/src/main/resources/webapp/src/app/core/core.module.ts
index f815c70..411849c 100644
--- a/services/self-service/src/main/resources/webapp/src/app/core/core.module.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/core/core.module.ts
@@ -40,6 +40,7 @@
import { DataengineConfigurationService } from './services/dataengineConfiguration.service';
import { StorageService } from './services/storage.service';
import { ProjectService } from './services/project.service';
+import { LegionDeploymentService } from './services/legion-deployment.service';
import { EndpointService } from './services/endpoint.service';
import { UserAccessKeyService } from './services/userAccessKey.service';
@@ -82,6 +83,7 @@
StorageService,
ProjectService,
EndpointService,
+ LegionDeploymentService,
UserAccessKeyService,
{ provide: MatDialogRef, useValue: {} },
diff --git a/services/self-service/src/main/resources/webapp/src/app/core/services/applicationServiceFacade.service.ts b/services/self-service/src/main/resources/webapp/src/app/core/services/applicationServiceFacade.service.ts
index 121188e..e6583f6 100644
--- a/services/self-service/src/main/resources/webapp/src/app/core/services/applicationServiceFacade.service.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/core/services/applicationServiceFacade.service.ts
@@ -75,6 +75,7 @@
private static readonly DOWNLOAD_REPORT = 'download_report';
private static readonly SETTINGS = 'settings';
private static readonly PROJECT = 'project';
+ private static readonly ODAHU = 'odahu';
private static readonly ENDPOINT = 'endpoint';
private static readonly ENDPOINT_CONNECTION = 'endpoint_connection';
@@ -609,6 +610,24 @@
null);
}
+ public createOdahuCluster(data): Observable<any> {
+ return this.buildRequest(HTTPMethod.POST,
+ this.requestRegistry.Item(ApplicationServiceFacade.ODAHU),
+ data);
+ }
+
+ public getOdahuList(): Observable<any> {
+ return this.buildRequest(HTTPMethod.GET,
+ this.requestRegistry.Item(ApplicationServiceFacade.ODAHU),
+ null);
+ }
+
+ public odahuStartStop(data, params): Observable<any> {
+ return this.buildRequest(HTTPMethod.POST,
+ this.requestRegistry.Item(ApplicationServiceFacade.ODAHU) + `/${params}`,
+ data);
+ }
+
private setupRegistry(): void {
this.requestRegistry = new Dictionary<string>();
@@ -688,6 +707,9 @@
this.requestRegistry.Add(ApplicationServiceFacade.PROJECT, '/api/project');
this.requestRegistry.Add(ApplicationServiceFacade.ENDPOINT, '/api/endpoint');
this.requestRegistry.Add(ApplicationServiceFacade.ENDPOINT_CONNECTION, '/api/endpoint/url/');
+
+ // odahu
+ this.requestRegistry.Add(ApplicationServiceFacade.ODAHU, '/api/odahu');
}
private buildRequest(method: HTTPMethod, url_path: string, body: any, opt?) {
diff --git a/services/self-service/src/main/resources/webapp/src/app/core/services/index.ts b/services/self-service/src/main/resources/webapp/src/app/core/services/index.ts
index 0068a29..395c667 100644
--- a/services/self-service/src/main/resources/webapp/src/app/core/services/index.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/core/services/index.ts
@@ -38,3 +38,4 @@
export * from './storage.service';
export * from './project.service';
export * from './endpoint.service';
+export * from './legion-deployment.service';
diff --git a/services/self-service/src/main/resources/webapp/src/app/core/services/legion-deployment.service.ts b/services/self-service/src/main/resources/webapp/src/app/core/services/legion-deployment.service.ts
new file mode 100644
index 0000000..1b0872e
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/core/services/legion-deployment.service.ts
@@ -0,0 +1,35 @@
+import { Injectable } from '@angular/core';
+import {Observable} from 'rxjs';
+import { map, catchError } from 'rxjs/operators';
+import { ApplicationServiceFacade } from './applicationServiceFacade.service';
+import { ErrorUtils } from '../util';
+
+@Injectable()
+
+export class LegionDeploymentService {
+ constructor(private applicationServiceFacade: ApplicationServiceFacade) { }
+
+ public createOdahuNewCluster(data): Observable<{}> {
+ return this.applicationServiceFacade
+ .createOdahuCluster(data)
+ .pipe(
+ map(response => response),
+ catchError(ErrorUtils.handleServiceError));
+ }
+
+ public getOduhuClustersList(): Observable<{}> {
+ return this.applicationServiceFacade
+ .getOdahuList()
+ .pipe(
+ map(response => response),
+ catchError(ErrorUtils.handleServiceError));
+ }
+
+ public odahuAction(data, action) {
+ return this.applicationServiceFacade
+ .odahuStartStop(data, action)
+ .pipe(
+ map(response => response),
+ catchError(ErrorUtils.handleServiceError));
+ }
+}
diff --git a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.ts b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.ts
index 1856688..29914e5 100644
--- a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting-grid/reporting-grid.component.ts
@@ -81,7 +81,7 @@
resetFiltering(): void {
this.filteredReportData.defaultConfigurations();
- this.filter.nativeElement.value = ''
+ this.filter.nativeElement.value = '';
this.filterReport.emit(this.filteredReportData);
this.resetRangePicker.emit(true);
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts
index 3433383..958eb23 100644
--- a/services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/reporting/reporting.component.ts
@@ -96,6 +96,11 @@
if (this.PROVIDER) {
this.rebuildBillingReport();
}
+ } else {
+ this.PROVIDER = 'azure';
+ if (this.PROVIDER) {
+ this.rebuildBillingReport();
+ }
}
}, e => {
this.PROVIDER = 'azure';
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resources-list/computational-resources-list.component.scss b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resources-list/computational-resources-list.component.scss
index a4b825e..c47cc43 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resources-list/computational-resources-list.component.scss
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/computational/computational-resources-list/computational-resources-list.component.scss
@@ -110,7 +110,7 @@
}
}
@media screen and (max-width: 1520px) {
- .resources,
+ .resources,
managment {
.source {
.resource-wrap {
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.html
index 4977b22..de503f3 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.html
@@ -18,11 +18,24 @@
-->
<div class="detail-dialog" id="dialog-box">
- <header class="dialog-header header-white">
+ <header class="dialog-header header-white" >
<button type="button" class="close" (click)="dialogRef.close()">×</button>
</header>
<div class="dialog-content">
- <div *ngIf="data">
+ <div *ngIf="data.legion">
+ <table class="detail-header">
+ <tr>
+ <td>{{legion.name}}</td>
+ <td>
+ <span class="status" ngClass="{{legion.status || ''}}">
+ {{legion.status}}
+ </span>
+ </td>
+ <td>{{legion.shape}}</td>
+ </tr>
+ </table>
+ </div>
+ <div *ngIf="data.notebook">
<table class="detail-header">
<tr>
<td>{{notebook.template_name}}</td>
@@ -129,5 +142,26 @@
</div>
</div>
</div>
+
+ <div class="legion-info" *ngIf="data.legion">
+ <div class="content-box">
+ <div class="detail-info" *ngIf="!legion.error_message">
+ <div class="links_block">
+ <div *ngFor="let url of legion.url" class="odahu-links">
+ <div class="odahu-link">
+ <span class="description">{{url.description }}: </span>
+ <a class="ellipsis" matTooltip="{{ url.url}}" matTooltipPosition="above" href="{{ url.url}}"
+ target="_blank">{{ url.url}}
+ </a>
+ </div>
+ <div class="grafana" *ngIf="url.description === 'Grafana'">
+ <div><span>Gafana user: </span><span class="creds">{{legion.bucket_name}}</span></div>
+ <div><span>Gafana password: </span><span class="creds">{{legion.shared_bucket_name}}</span></div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
</div>
</div>
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.scss b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.scss
index 2849ce0..f4d17ef 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.scss
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.scss
@@ -26,17 +26,42 @@
}
.links_block {
+ max-height: 300px;
+ overflow-y: auto;
- >p {
- display: flex;
+ .odahu-links:not(:last-child){
+ margin-bottom: 20px;
+ }
- .description {
- white-space: nowrap;
- padding-left: 7px;
- font-weight: 600;
- color: $blue-grey-color;
+ .grafana{
+ white-space: nowrap;
+ padding-left: 30px;
+ color: $blue-grey-color;
+
+ div{
+ display: flex;
+
+ span{
+ font-size: 13px;
+ font-weight: 500;
+
+ &.creds{
+ font-weight: 600;
+ }
+ }
}
}
+
+ >p, .odahu-link {
+ display: flex;
+
+ .description {
+ white-space: nowrap;
+ padding-left: 7px;
+ font-weight: 600;
+ color: $blue-grey-color;
+ }
+ }
}
.checkbox-group {
@@ -73,7 +98,6 @@
}
span {
-
.danger_color {
position: absolute;
bottom: -16px;
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.ts
index db097f3..c2f607a 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/exploratory/detail-dialog/detail-dialog.component.ts
@@ -35,7 +35,7 @@
export class DetailDialogComponent implements OnInit {
readonly DICTIONARY = DICTIONARY;
- readonly PROVIDER = this.data.cloud_provider;
+ public PROVIDER: string;
notebook: any;
upTimeInHours: number;
upTimeSince: string = '';
@@ -43,6 +43,7 @@
config: Array<{}> = [];
public configurationForm: FormGroup;
+ private legion: any;
@ViewChild('configurationNode', { static: false }) configuration;
@@ -53,13 +54,19 @@
public dialogRef: MatDialogRef<DetailDialogComponent>,
public toastr: ToastrService
) {
- this.notebook = data;
+ if(data.notebook) {
+ this.notebook = data.notebook;
+ this.PROVIDER = this.data.notebook.cloud_provider;
+ }
+
+ if(data.legion) {
+ this.legion = data.legion;
+ this.PROVIDER = this.data.legion.cloud_provider || 'azure';
+ }
}
ngOnInit() {
- this.notebook;
-
- if (this.notebook) {
+ if (this.notebook) {
this.tooltip = false;
this.upTimeInHours = (this.notebook.time) ? DateUtils.diffBetweenDatesInHours(this.notebook.time) : 0;
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html
index 965f9d8..ea0c65f 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.html
@@ -105,8 +105,9 @@
[@detailExpand]="element == expandedElement ? 'expanded' : 'collapsed'" sticky>
<tr *ngFor="let element of element.exploratory; let i = index" class="element-row mat-row">
- <td class="name-col" (click)="printDetailEnvironmentModal(element)">
- <span matTooltip="{{ element.name }}" matTooltipPosition="above">{{ element.name }}</span>
+ <td class="name-col">
+ <span *ngIf="element.shape !== 'odahu cluster'" matTooltip="{{ element.name }}" matTooltipPosition="above" (click)="printDetailEnvironmentModal(element)">{{ element.name }}</span>
+ <span *ngIf="element.shape === 'odahu cluster'" matTooltip="{{ element.name }}" matTooltipPosition="above" (click)="printDetailLegionModal(element)">{{ element.name }}</span>
</td>
<td class="status-col status" ngClass="{{ element.status.toLowerCase() || ''}}">
{{element.status | underscoreless }}
@@ -122,7 +123,7 @@
matTooltipPosition="above">
{{ element.tags.custom_tag }}
</mat-chip>
- <mat-chip matTooltip="User tag: {{ element.tags.user_tag }}" matTooltipPosition="above">
+ <mat-chip *ngIf="element.tags.user_tag" matTooltip="User tag: {{ element.tags.user_tag }}" matTooltipPosition="above">
{{ element.tags.user_tag }}
</mat-chip>
<mat-chip matTooltip="Project tag: {{ element.tags.project_tag }}" matTooltipPosition="above">
@@ -132,14 +133,14 @@
</td>
<td class="resources-col">
- <computational-resources-list [resources]="element.resources" [environment]="element"
+ <computational-resources-list *ngIf="element.shape !== 'odahu cluster'" [resources]="element.resources" [environment]="element"
(buildGrid)="buildGrid($event)">
</computational-resources-list>
</td>
<td *ngIf="healthStatus?.billingEnabled" class="cost-col">
<span class="total_cost">{{ element.cost || 'N/A' }} {{ element.currency_code || '' }}</span>
<span (click)="element.billing && printCostDetails(element)" class="currency_details"
- [ngClass]="{ 'not-allowed' : !element.billing }">
+ [ngClass]="{ 'not-allowed' : !element.billing || element.shape === 'odahu cluster' }">
<i class="material-icons">help_outline</i>
</span>
</td>
@@ -148,11 +149,12 @@
<span #settings (click)="actions.toggle($event, settings)" class="actions"
[ngClass]="{ 'disabled': element.status.toLowerCase() === 'creating'
|| (element.image === 'docker.dlab-superset' && element.status !== 'running' && element.status !== 'stopped')
- || (element.image === 'docker.dlab-jupyterlab' && element.status !== 'running' && element.status !== 'stopped') }">
+ || (element.image === 'docker.dlab-jupyterlab' && element.status !== 'running' && element.status !== 'stopped')
+ }">
</span>
<bubble-up #actions class="list-menu" position="bottom-left" alternative="top-left">
- <ul class="list-unstyled">
+ <ul class="list-unstyled" *ngIf="element.shape !== 'odahu cluster'">
<div class="active-items" *ngIf="element.status.toLowerCase() !== 'failed'
&& element.status !== 'terminating'
&& element.status !== 'terminated'
@@ -214,6 +216,29 @@
</a>
</li>
</ul>
+ <ul class="list-unstyled" *ngIf="element.shape === 'odahu cluster'">
+ <li class="project-seting-item" *ngIf="element.status === 'stopped'" (click)="odahuAction(element, 'start')">
+ <i class="material-icons">play_circle_outline</i>
+ <a class="action">
+ Start
+ </a>
+ </li>
+ <li class="project-seting-item" *ngIf="element.status === 'running'" (click)="odahuAction(element, 'stop')">
+ <i class="material-icons">pause_circle_outline</i>
+ <a class="action" >
+ Stop
+ </a>
+ </li>
+ <li class="project-seting-item"
+ [ngClass]="{'disabled': element.status === 'stopped' || element.status.toLowerCase() === 'stopping' || element.status.toLowerCase() === 'starting'}"
+ *ngIf="element.status === element.status !== 'terminated' && element.status !== 'terminating'" (click)="odahuAction(element, 'terminate')"
+ >
+ <i class="material-icons">phonelink_off</i>
+ <a class="action">
+ Terminate
+ </a>
+ </li>
+ </ul>
</bubble-up>
</td>
</tr>
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.scss b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.scss
index e09ff3e..1657a56 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.scss
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.scss
@@ -392,6 +392,11 @@
text-align: center;
}
+.legion-icon{
+ vertical-align: middle;
+ margin-left: 10px;
+}
+
.content-row{
background-clip: padding-box;
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.ts b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.ts
index 3052d84..861981e 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.component.ts
@@ -23,7 +23,7 @@
import { ToastrService } from 'ngx-toastr';
import { MatDialog } from '@angular/material/dialog';
-import { UserResourceService } from '../../core/services';
+import {LegionDeploymentService, UserResourceService} from '../../core/services';
import { ExploratoryModel, Exploratory } from './resources-grid.model';
import { FilterConfigurationModel } from './filter-configuration.model';
@@ -40,6 +40,7 @@
import { DICTIONARY } from '../../../dictionary/global.dictionary';
import {ProgressBarService} from '../../core/services/progress-bar.service';
+import {OdahuActionDialogComponent} from '../../shared/modal-dialog/odahu-action-dialog';
import {ComputationModel} from '../computational/computational-resource.model';
import {NotebookModel} from '../exploratory/notebook.model';
@@ -95,6 +96,7 @@
private userResourceService: UserResourceService,
private dialog: MatDialog,
private progressBarService: ProgressBarService,
+ private legionDeploymentService: LegionDeploymentService,
) { }
ngOnInit(): void {
@@ -144,7 +146,7 @@
public isResourcesInProgress(notebook) {
const env = this.getResourceByName(notebook.name, notebook.project);
- if (env && env.resources.length) {
+ if (env && env.resources && env.resources.length) {
return env.resources.filter(item => (item.status !== 'failed' && item.status !== 'terminated'
&& item.status !== 'running' && item.status !== 'stopped')).length > 0;
}
@@ -173,7 +175,12 @@
}
public printDetailEnvironmentModal(data): void {
- this.dialog.open(DetailDialogComponent, { data: data, panelClass: 'modal-lg' })
+ this.dialog.open(DetailDialogComponent, { data: {notebook: data}, panelClass: 'modal-lg' })
+ .afterClosed().subscribe(() => this.buildGrid());
+ }
+
+ public printDetailLegionModal(data): void {
+ this.dialog.open(DetailDialogComponent, { data: {legion: data}, panelClass: 'modal-lg' })
.afterClosed().subscribe(() => this.buildGrid());
}
@@ -219,7 +226,7 @@
private getResourceByName(notebook_name: string, project_name: string) {
return this.getEnvironmentsListCopy().filter(environments => environments.project === project_name)
.map(env => env.exploratory.find(({ name }) => name === notebook_name))
- .filter(name => !!name)[0];
+ .filter(notebook_name => !!notebook_name)[0];
}
private getEnvironmentsListCopy() {
@@ -234,11 +241,11 @@
if (shapes.indexOf(item.shape) === -1) shapes.push(item.shape);
if (statuses.indexOf(item.status) === -1) statuses.push(item.status);
statuses.sort(SortUtils.statusSort);
-
+ if (item.resources) {
item.resources.map((resource: any) => {
if (resources.indexOf(resource.status) === -1) resources.push(resource.status);
resources.sort(SortUtils.statusSort);
- });
+ }); }
}));
this.filterConfiguration = new FilterConfigurationModel('', statuses, shapes, resources, '', '');
@@ -249,7 +256,10 @@
let filteredData = this.getEnvironmentsListCopy();
const containsStatus = (list, selectedItems) => {
- return list.filter((item: any) => { if (selectedItems.indexOf(item.status) !== -1) return item; });
+ if (list && list.length) {
+ return list.filter((item: any) => { if (selectedItems.indexOf(item.status) !== -1) return item; });
+ }
+ return list;
};
if (filteredData.length) this.filtering = true;
@@ -264,7 +274,6 @@
const isName = item.name.toLowerCase().indexOf(config.name.toLowerCase()) !== -1;
const isStatus = config.statuses.length > 0 ? (config.statuses.indexOf(item.status) !== -1) : (config.type !== 'active');
const isShape = config.shapes.length > 0 ? (config.shapes.indexOf(item.shape) !== -1) : true;
-
const modifiedResources = containsStatus(item.resources, config.resources);
let isResources = config.resources.length > 0 ? (modifiedResources.length > 0) : true;
@@ -352,4 +361,15 @@
.subscribe((result) => { },
(error) => console.log('UPDATE USER PREFERENCES ERROR ', error));
}
+
+ private odahuAction(element: any, action: string) {
+ this.dialog.open(OdahuActionDialogComponent, {data: {type: action, item: element}, panelClass: 'modal-sm'})
+ .afterClosed().subscribe(result => {
+ result && this.legionDeploymentService.odahuAction(element, action).subscribe(v =>
+ this.buildGrid(),
+ error => this.toastr.error(`Odahu cluster ${action} failed!`, 'Oops!')
+ ) ;
+ }, error => this.toastr.error(error.message || `Odahu cluster ${action} failed!`, 'Oops!')
+ );
+ }
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts
index 14a2824..7d40063 100644
--- a/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/resources/resources-grid/resources-grid.model.ts
@@ -55,11 +55,12 @@
public static loadEnvironments(data: Array<any>) {
if (data) {
return data.map((value) => {
- return {
- project: value.project,
- exploratory: value.exploratory.map(el => {
- const provider = el.cloud_provider.toLowerCase();
- return new ExploratoryModel(
+ const exploratory = value.exploratory.map(el => {
+ let provider;
+ if (el.cloud_provider) {
+ provider = el.cloud_provider.toLowerCase();
+ }
+ return new ExploratoryModel(
provider,
el.exploratory_name,
el.template_name,
@@ -73,13 +74,13 @@
el.private_ip,
el.exploratory_user,
el.exploratory_pass,
- value.shared[el.endpoint][DICTIONARY[provider].bucket_name],
- value.shared[el.endpoint][DICTIONARY[provider].shared_bucket_name],
- el.error_message,
- el[DICTIONARY[provider].billing.cost],
- el[DICTIONARY[provider].billing.currencyCode],
- el.billing,
- el.libs,
+ value.shared[el.endpoint][DICTIONARY[provider].bucket_name],
+ value.shared[el.endpoint][DICTIONARY[provider].shared_bucket_name],
+ el.error_message,
+ el[DICTIONARY[provider].billing.cost],
+ el[DICTIONARY[provider].billing.currencyCode],
+ el.billing,
+ el.libs,
value.shared[el.endpoint][DICTIONARY[provider].user_storage_account_name],
value.shared[el.endpoint][DICTIONARY[provider].shared_storage_account_name],
value.shared[el.endpoint][DICTIONARY[provider].datalake_name],
@@ -87,8 +88,50 @@
value.shared[el.endpoint][DICTIONARY[provider].datalake_shared_directory_name],
el.project,
el.endpoint,
- el.tags,
- )})
+ el.tags
+ );
+ });
+
+ const odahu = value.odahu.map(el => {
+ let provider;
+ if (el.cloud_provider) {
+ provider = el.cloud_provider.toLowerCase();
+ } else {
+ provider = 'azure';
+ }
+ return new ExploratoryModel(
+ provider,
+ el.name,
+ el.template_name,
+ el.image,
+ el.status.toLowerCase(),
+ 'odahu cluster',
+ [],
+ el.up_time,
+ el.urls,
+ value.shared[el.endpoint].edge_node_ip,
+ el.private_ip,
+ el.exploratory_user,
+ el.exploratory_pass,
+ el.grafana_admin,
+ el.grafana_pass,
+ el.error_message,
+ el[DICTIONARY[provider].billing.cost],
+ el[DICTIONARY[provider].billing.currencyCode],
+ [],
+ [],
+ '',
+ '',
+ '',
+ '',
+ '',
+ el.project,
+ el.endpoint,
+ el.tags
+ )});
+ return {
+ project: value.project,
+ exploratory: [...exploratory, ...odahu]
};
});
}
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/edge-action-dialog/index.ts b/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/edge-action-dialog/index.ts
index fd6e98a..c91d503 100644
--- a/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/edge-action-dialog/index.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/edge-action-dialog/index.ts
@@ -21,7 +21,7 @@
import { CommonModule } from '@angular/common';
import { EdgeActionDialogComponent } from './edge-action-dialog.component';
import { MaterialModule } from '../../material.module';
-import {FormsModule} from "@angular/forms";
+import {FormsModule} from '@angular/forms';
export * from './edge-action-dialog.component';
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/odahu-action-dialog/index.ts b/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/odahu-action-dialog/index.ts
new file mode 100644
index 0000000..bfc5420
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/odahu-action-dialog/index.ts
@@ -0,0 +1,15 @@
+import { NgModule } from '@angular/core';
+import { CommonModule } from '@angular/common';
+import { OdahuActionDialogComponent } from './odahu-action-dialog.component';
+import { MaterialModule } from '../../material.module';
+import {FormsModule} from '@angular/forms';
+
+export * from './odahu-action-dialog.component';
+
+@NgModule({
+ imports: [CommonModule, MaterialModule, FormsModule],
+ declarations: [OdahuActionDialogComponent],
+ entryComponents: [OdahuActionDialogComponent],
+ exports: [OdahuActionDialogComponent]
+})
+export class OdahuActionDialogModule {}
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/odahu-action-dialog/odahu-action-dialog.component.ts b/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/odahu-action-dialog/odahu-action-dialog.component.ts
new file mode 100644
index 0000000..921ff99
--- /dev/null
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/modal-dialog/odahu-action-dialog/odahu-action-dialog.component.ts
@@ -0,0 +1,50 @@
+import { Component, Inject } from '@angular/core';
+import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
+
+
+@Component({
+ selector: 'edge-action-dialog',
+ template: `
+ <div id="dialog-box">
+ <header class="dialog-header">
+ <h4 class="modal-title"><span class="action">{{data.type | titlecase}}</span> Odahu cluster</h4>
+ <button type="button" class="close" (click)="dialogRef.close()">×</button>
+ </header>
+ <div mat-dialog-content class="content message mat-dialog-content">
+ <h3>Odahu cluster <span class="strong">{{data.item.name}} </span>will be {{label[data.type]}}</h3>
+ <p class="m-top-20 action-text"><span class="strong">Do you want to proceed?</span></p>
+
+ <div class="text-center m-top-30 m-bott-30">
+ <button type="button" class="butt" mat-raised-button (click)="dialogRef.close()">No</button>
+ <button type="button" class="butt butt-success" mat-raised-button (click)="dialogRef.close(true)">Yes</button>
+ </div>
+ </div>
+ </div>
+ `,
+ styles: [`
+ .content { color: #718ba6; padding: 20px 50px; font-size: 14px; font-weight: 400; margin: 0; }
+ .info .confirm-dialog { color: #607D8B; }
+ header { display: flex; justify-content: space-between; color: #607D8B; }
+ h3 { font-weight: 300; }
+ header h4 i { vertical-align: bottom; }
+ header a i { font-size: 20px; }
+ header a:hover i { color: #35afd5; cursor: pointer; }
+ .action{text-transform: capitalize}
+ .action-text { text-align: center; }
+ label { font-size: 15px; font-weight: 500; font-family: "Open Sans",sans-serif; cursor: pointer; display: flex; align-items: center;}
+ label input {margin-top: 2px; margin-right: 5px;}
+ `]
+})
+
+export class OdahuActionDialogComponent {
+ public label = {
+ stop: 'stopped',
+ start: 'started',
+ terminate: 'terminated',
+ };
+
+ constructor(
+ public dialogRef: MatDialogRef<OdahuActionDialogComponent>,
+ @Inject(MAT_DIALOG_DATA) public data: any) {
+ }
+}
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/navbar/index.ts b/services/self-service/src/main/resources/webapp/src/app/shared/navbar/index.ts
index 7a6f2d4..2796b61 100644
--- a/services/self-service/src/main/resources/webapp/src/app/shared/navbar/index.ts
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/navbar/index.ts
@@ -25,7 +25,8 @@
import { NavbarComponent } from './navbar.component';
import { NotificationDialogModule } from '../modal-dialog/notification-dialog';
-import {EdgeActionDialogModule} from "../modal-dialog/edge-action-dialog";
+import {EdgeActionDialogModule} from '../modal-dialog/edge-action-dialog';
+import {OdahuActionDialogModule} from '../modal-dialog/odahu-action-dialog';
export * from './navbar.component';
@@ -36,6 +37,7 @@
MaterialModule,
NotificationDialogModule,
EdgeActionDialogModule,
+ OdahuActionDialogModule,
ProgressDialogModule,
BubbleModule,
],
diff --git a/services/self-service/src/main/resources/webapp/src/app/shared/navbar/navbar.component.html b/services/self-service/src/main/resources/webapp/src/app/shared/navbar/navbar.component.html
index 9485a87..1b6dc05 100644
--- a/services/self-service/src/main/resources/webapp/src/app/shared/navbar/navbar.component.html
+++ b/services/self-service/src/main/resources/webapp/src/app/shared/navbar/navbar.component.html
@@ -88,6 +88,11 @@
<span *ngIf="isExpanded; else projects">Projects</span>
<ng-template #projects><i class="material-icons">dns</i></ng-template>
</a>
+ <a class="sub-nav-item" [style.margin-left.px]="isExpanded ? '30' : '0'" [routerLink]="['/odahu']"
+ [routerLinkActive]="['active']" [routerLinkActiveOptions]="{exact:true}">
+ <span *ngIf="isExpanded; else legion">Odahu deployment</span>
+ <ng-template #legion><i class="material-icons">get_app</i></ng-template>
+ </a>
<a class="sub-nav-item" [style.margin-left.px]="isExpanded ? '30' : '0'"
[routerLink]="['/environment_management']" [routerLinkActive]="['active']"
[routerLinkActiveOptions]="{exact:true}">
diff --git a/services/self-service/src/main/resources/webapp/src/assets/styles/_dialogs.scss b/services/self-service/src/main/resources/webapp/src/assets/styles/_dialogs.scss
index c6f8fe8..595f36c 100644
--- a/services/self-service/src/main/resources/webapp/src/assets/styles/_dialogs.scss
+++ b/services/self-service/src/main/resources/webapp/src/assets/styles/_dialogs.scss
@@ -340,6 +340,12 @@
font-weight: 500;
max-height: 200px;
overflow: auto;
+
+.disabled {
+ opacity: 0.4;
+ cursor: not-allowed;
+ pointer-events: none;
+
}
@media screen and (max-width: 1280px) {
diff --git a/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResourceTest.java b/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResourceTest.java
index b92335e..e31b1ed 100644
--- a/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResourceTest.java
+++ b/services/self-service/src/test/java/com/epam/dlab/backendapi/resources/InfrastructureInfoResourceTest.java
@@ -37,7 +37,16 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyBoolean;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.refEq;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
+import static org.mockito.Mockito.when;
public class InfrastructureInfoResourceTest extends TestBase {
@@ -159,7 +168,7 @@
@Test
public void getUserResourcesWithException() {
doThrow(new DlabException("Could not load list of provisioned resources for user"))
- .when(infrastructureInfoService).getUserResources(anyString());
+ .when(infrastructureInfoService).getUserResources(any(UserInfo.class));
final Response response = resources.getJerseyTest()
.target("/infrastructure/info")
.request()
@@ -169,7 +178,7 @@
assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, response.getStatus());
assertEquals(MediaType.APPLICATION_JSON, response.getHeaderString(HttpHeaders.CONTENT_TYPE));
- verify(infrastructureInfoService).getUserResources(USER.toLowerCase());
+ verify(infrastructureInfoService).getUserResources(any(UserInfo.class));
verifyNoMoreInteractions(infrastructureInfoService);
}
