[YUNIKORN-2385] Update website for v1.5 release
diff --git a/doap_YuniKorn.rdf b/doap_YuniKorn.rdf
index f803de8..27945fa 100644
--- a/doap_YuniKorn.rdf
+++ b/doap_YuniKorn.rdf
@@ -36,6 +36,13 @@
<category rdf:resource="https://projects.apache.org/category/cloud" />
<release>
<Version>
+ <name>YuniKorn 1.5.0</name>
+ <created>2024-03-14</created>
+ <revision>1.5.0</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
<name>YuniKorn 1.4.0</name>
<created>2023-11-20</created>
<revision>1.4.0</revision>
diff --git a/docusaurus.config.js b/docusaurus.config.js
index 782d030..d50e714 100644
--- a/docusaurus.config.js
+++ b/docusaurus.config.js
@@ -49,7 +49,7 @@
announcementBar: {
id: 'new_release',
content:
- '1.4.0 has been released, check the <a href="/community/download">DOWNLOADS</a>.',
+ '1.5.0 has been released, check the <a href="/community/download">DOWNLOADS</a>.',
backgroundColor: '#fafbfc',
textColor: '#091E42',
},
diff --git a/package.json b/package.json
index 825aca1..ba2b536 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,7 @@
"dependencies": {
"@docusaurus/core": "2.4.1",
"@docusaurus/preset-classic": "2.4.1",
- "@docusaurus/theme-search-algolia": "^2.4.1",
+ "@docusaurus/theme-search-algolia": "^3.1.1",
"@mdx-js/react": "^1.5.8",
"clsx": "^1.1.1",
"node": "^18.8.0",
diff --git a/src/pages/community/download.md b/src/pages/community/download.md
index 54b938f..de81b23 100644
--- a/src/pages/community/download.md
+++ b/src/pages/community/download.md
@@ -31,11 +31,12 @@
We publish prebuilt docker images for everyone's convenience.
-The latest release of Apache YuniKorn is v1.4.0.
+The latest release of Apache YuniKorn is v1.5.0.
-| Version | Release date | Source download | Docker images | Release notes |
-|---------|--------------|--------------------------------------------------------|---------------------------------------------------------------------------------------------|---------------------|
-| v1.4.0 | 2023-11-20 | [Download](https://downloads.apache.org/yunikorn/1.4.0/apache-yunikorn-1.4.0-src.tar.gz)<br/>[Checksum](https://downloads.apache.org/yunikorn/1.4.0/apache-yunikorn-1.4.0-src.tar.gz.sha512) & [Signature](https://downloads.apache.org/yunikorn/1.4.0/apache-yunikorn-1.4.0-src.tar.gz.asc) | [scheduler](https://hub.docker.com/layers/apache/yunikorn/scheduler-1.4.0/images/sha256-d013be8e3ad7eb8e51ce23951e6899a4b74088e52c3767f3fcc7efcdcc0904f5?context=explore)<br/>[admission-controller](https://hub.docker.com/layers/apache/yunikorn/admission-1.4.0/images/sha256-d93cd7cb480d8bd0ae829d88484b5c8b8f89c843dd0ea48694a636cc0bb00e07?context=explore)<br/>[web](https://hub.docker.com/layers/apache/yunikorn/web-1.4.0/images/sha256-60a732eb04a9690214d2d2f852058a501585091901fb9c0faf66a378e710d452?context=explore)<br/>[scheduler plugin](https://hub.docker.com/layers/apache/yunikorn/scheduler-plugin-1.4.0/images/sha256-7a82c87f4f6caf950529478851f0aaa5da2b225668325ee50b7422c477804e02?context=explore) | [Announcement](/release-announce/1.4.0) |
+| Version | Release date | Source download | Docker images | Release notes |
+|---------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------|-----------------------------------------|
+| 1.5.0 | 2024-03-14 | [Download](https://downloads.apache.org/yunikorn/1.5.0/apache-yunikorn-1.5.0-src.tar.gz)<br/>[Checksum](https://downloads.apache.org/yunikorn/1.5.0/apache-yunikorn-1.5.0-src.tar.gz.sha512) & [Signature](https://downloads.apache.org/yunikorn/1.5.0/apache-yunikorn-1.5.0-src.tar.gz.asc) | [scheduler](https://hub.docker.com/layers/apache/yunikorn/scheduler-1.5.0/images/sha256-9cefd0df164b9c4d39f9e10b010eaf7d8f89b130de1648e94f75b9b95d300a00?context=explore)<br/>[admission-controller](https://hub.docker.com/layers/apache/yunikorn/admission-1.5.0/images/sha256-28f63ec17ac96faa08d79a5e133a2728bc339319f4b71d481f3c0e6477944697?context=explore)<br/>[web](https://hub.docker.com/layers/apache/yunikorn/web-1.5.0/images/sha256-ccdc9d1e4cbc36037e181495f48e569bfc43a24f65efbd52d8ddc7183f1f3e17?context=explore)<br/>[scheduler plugin](https://hub.docker.com/layers/apache/yunikorn/scheduler-plugin-1.5.0/images/sha256-01f1a6a8aab81aa3b385e4d439a1b919174626475bd8939af8c944f962170d05?context=explore) | [Announcement](/release-announce/1.5.0) |
+| v1.4.0 | 2023-11-20 | [Download](https://archive.apache.org/dist/yunikorn/1.4.0/apache-yunikorn-1.4.0-src.tar.gz)<br/>[Checksum](https://archive.apache.org/dist/yunikorn/1.4.0/apache-yunikorn-1.4.0-src.tar.gz.sha512) & [Signature](https://archive.apache.org/dist/yunikorn/1.4.0/apache-yunikorn-1.4.0-src.tar.gz.asc) | [scheduler](https://hub.docker.com/layers/apache/yunikorn/scheduler-1.4.0/images/sha256-d013be8e3ad7eb8e51ce23951e6899a4b74088e52c3767f3fcc7efcdcc0904f5?context=explore)<br/>[admission-controller](https://hub.docker.com/layers/apache/yunikorn/admission-1.4.0/images/sha256-d93cd7cb480d8bd0ae829d88484b5c8b8f89c843dd0ea48694a636cc0bb00e07?context=explore)<br/>[web](https://hub.docker.com/layers/apache/yunikorn/web-1.4.0/images/sha256-60a732eb04a9690214d2d2f852058a501585091901fb9c0faf66a378e710d452?context=explore)<br/>[scheduler plugin](https://hub.docker.com/layers/apache/yunikorn/scheduler-plugin-1.4.0/images/sha256-7a82c87f4f6caf950529478851f0aaa5da2b225668325ee50b7422c477804e02?context=explore) | [Announcement](/release-announce/1.4.0) |
| v1.3.0 | 2023-06-12 | [Download](https://archive.apache.org/dist/yunikorn/1.3.0/apache-yunikorn-1.3.0-src.tar.gz)<br/>[Checksum](https://archive.apache.org/dist/yunikorn/1.3.0/apache-yunikorn-1.3.0-src.tar.gz.sha512) & [Signature](https://archive.apache.org/dist/yunikorn/1.3.0/apache-yunikorn-1.3.0-src.tar.gz.asc) | [scheduler](https://hub.docker.com/layers/apache/yunikorn/scheduler-1.3.0/images/sha256-99a1973728c6684b1da7631dbf015daa1dbf519dbab1ffc8b23fccdfa7ffd0c5?context=explore)<br/>[admission-controller](https://hub.docker.com/layers/apache/yunikorn/admission-1.3.0/images/sha256-3fb41eafcb16ec709879301f0f1cf5ffd18d95e6bb266b20e2971c39c6f6fc94?context=explore)<br/>[web](https://hub.docker.com/layers/apache/yunikorn/web-1.3.0/images/sha256-47c1ff0b58c2c0833bf8662065f7517b8e235dbc2197a9511549ec2ee4b31969?context=explore)<br/>[scheduler plugin](https://hub.docker.com/layers/apache/yunikorn/scheduler-plugin-1.3.0/images/sha256-c3c564033dd8ea07d2f7c5fe272be43b8eba7e7b115ac9b5bee4cf8cae681cd9?context=explore) | [Announcement](/release-announce/1.3.0) |
| v1.2.0 | 2023-02-02 | [Download](https://archive.apache.org/dist/yunikorn/1.2.0/apache-yunikorn-1.2.0-src.tar.gz)<br/>[Checksum](https://archive.apache.org/dist/yunikorn/1.2.0/apache-yunikorn-1.2.0-src.tar.gz.sha512) & [Signature](https://archive.apache.org/dist/yunikorn/1.2.0/apache-yunikorn-1.2.0-src.tar.gz.asc) | [scheduler](https://hub.docker.com/layers/apache/yunikorn/scheduler-1.2.0/images/sha256-c3b1a7b2cfec3f3560415519278cc4d94748f0f60ee80dfaf23fcc22dbb8b8e5)<br/>[admission-controller](https://hub.docker.com/layers/apache/yunikorn/admission-1.2.0/images/sha256-7f18fcd080640974ae586d30eda009daf0ad93fa22ada66b0a337ac3fb46b7ac)<br/>[web](https://hub.docker.com/layers/apache/yunikorn/web-1.2.0/images/sha256-706a2895461623f20d4102f0132d66dade9c15edf5cad40065506a4d70c32576)<br/>[scheduler plugin](https://hub.docker.com/layers/apache/yunikorn/scheduler-plugin-1.2.0/images/sha256-d946495946b89d03f7a8f786702d4b350a93f74d52e50bebb6b2bbdcb8e911a4?context=explore) | [Announcement](/release-announce/1.2.0) |
diff --git a/src/pages/release-announce/1.5.0.md b/src/pages/release-announce/1.5.0.md
index a6212b8..61c33b8 100644
--- a/src/pages/release-announce/1.5.0.md
+++ b/src/pages/release-announce/1.5.0.md
@@ -32,7 +32,7 @@
Release manager: TingYao Huang
-Release date: 2024-03-01 (TBD)
+Release date: 2024-03-14
## Highlights
diff --git a/versioned_docs/version-1.5.0/api/cluster.md b/versioned_docs/version-1.5.0/api/cluster.md
new file mode 100644
index 0000000..35e98da
--- /dev/null
+++ b/versioned_docs/version-1.5.0/api/cluster.md
@@ -0,0 +1,79 @@
+---
+id: cluster
+title: Cluster
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Clusters
+
+Returns general information about the clusters managed by the YuniKorn Scheduler.
+The response includes build information about resource managers.
+
+**URL** : `/ws/v1/clusters`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+As an example, here is a response from a cluster with 1 resource manager.
+
+```json
+[
+ {
+ "startTime": 1697100824863892713,
+ "rmBuildInformation": [
+ {
+ "arch": "amd64",
+ "buildDate": "2023-09-04T18:11:43+0800",
+ "buildVersion": "latest",
+ "coreSHA": "0ecf24d2aad2",
+ "goVersion": "1.21",
+ "isPluginVersion": "false",
+ "rmId": "mycluster",
+ "shimSHA": "8b26c373b4b5",
+ "siSHA": "e7622cf54e95"
+ }
+ ],
+ "partition": "default",
+ "clusterName": "kubernetes"
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
diff --git a/versioned_docs/version-1.5.0/api/scheduler.md b/versioned_docs/version-1.5.0/api/scheduler.md
new file mode 100644
index 0000000..1ab0c64
--- /dev/null
+++ b/versioned_docs/version-1.5.0/api/scheduler.md
@@ -0,0 +1,2152 @@
+---
+id: scheduler
+title: Scheduler
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Overview
+
+The scheduler REST API returns information about various objects used by the YuniKorn Scheduler.
+
+Many of these APIs return collections of resources. Internally, all resources are represented as raw
+64-bit signed integer types. When interpreting responses from the REST API, resources of type `memory`
+are returned in units of bytes while resources of type `vcore` are returned in units of millicores
+(thousands of a core). All other resource types have no specific unit assigned.
+
+Under the `allocations` field in the response content for the app/node-related calls in the following spec, `placeholderUsed` refers to whether or not the allocation is a replacement for a placeholder. If true, `requestTime` is the creation time of its placeholder allocation, otherwise it's that of the allocation's ask. `allocationTime` is the creation time of the allocation, and `allocationDelay` is simply the difference between `allocationTime` and `requestTime`.
+
+## Partitions
+
+Returns general information and statistics about a partition.
+
+**URL** : `/ws/v1/partitions`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "clusterId": "mycluster",
+ "name": "default",
+ "state": "Active",
+ "lastStateTransitionTime": 1649167576110754000,
+ "capacity": {
+ "capacity": {
+ "ephemeral-storage": 188176871424,
+ "hugepages-1Gi": 0,
+ "hugepages-2Mi": 0,
+ "memory": 1000000000,
+ "pods": 330,
+ "vcore": 1000
+ },
+ "usedCapacity": {
+ "memory": 800000000,
+ "vcore": 500
+ },
+ "utilization": {
+ "memory": 80,
+ "vcore": 50
+ }
+ },
+ "nodeSortingPolicy": {
+ "type": "fair",
+ "resourceWeights": {
+ "memory": 1.5,
+ "vcore": 1.3
+ }
+ },
+ "applications": {
+ "New": 5,
+ "Pending": 5,
+ "total": 10
+ },
+ "totalContainers": 0,
+ "totalNodes": 2
+ },
+ {
+ "clusterId": "mycluster",
+ "name": "gpu",
+ "state": "Active",
+ "lastStateTransitionTime": 1649167576111236000,
+ "capacity": {
+ "capacity": {
+ "memory": 2000000000,
+ "vcore": 2000
+ },
+ "usedCapacity": {
+ "memory": 500000000,
+ "vcore": 300
+ },
+ "utilization": {
+ "memory": 25,
+ "vcore": 15
+ }
+ },
+ "nodeSortingPolicy": {
+ "type": "binpacking",
+ "resourceWeights": {
+ "memory": 0,
+ "vcore": 4.11
+ }
+ },
+ "applications": {
+ "New": 5,
+ "Running": 10,
+ "Pending": 5,
+ "total": 20
+ },
+ "totalContainers": 20,
+ "totalNodes": 5
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Queues
+
+### Partition queues
+
+Fetch all Queues associated with given Partition and displays general information about the queues like name, status, capacities and properties.
+The queues' hierarchy is kept in the response json.
+
+**URL** : `/ws/v1/partition/{partitionName}/queues`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+For the default queue hierarchy (only `root.default` leaf queue exists) a similar response to the following is sent back to the client:
+
+```json
+[
+ {
+ "queuename": "root",
+ "status": "Active",
+ "maxResource": {
+ "ephemeral-storage": 188176871424,
+ "hugepages-1Gi": 0,
+ "hugepages-2Mi": 0,
+ "memory": 8000000000,
+ "pods": 330,
+ "vcore": 8000
+ },
+ "guaranteedResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
+ "allocatedResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
+ "pendingResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
+ "isLeaf": "false",
+ "isManaged": "false",
+ "properties": {
+ "application.sort.policy": "stateaware"
+ },
+ "parent": "",
+ "template": {
+ "maxResource": {
+ "memory": 8000000000,
+ "vcore": 8000
+ },
+ "guaranteedResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
+ "properties": {
+ "application.sort.policy": "stateaware"
+ }
+ },
+ "partition": "default",
+ "children": [
+ {
+ "queuename": "root.default",
+ "status": "Active",
+ "maxResource": {
+ "memory": 8000000000,
+ "vcore": 8000
+ },
+ "guaranteedResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
+ "allocatedResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
+ "pendingResource": {
+ "memory": 54000000,
+ "vcore": 80
+ },
+ "isLeaf": "true",
+ "isManaged": "false",
+ "properties": {
+ "application.sort.policy": "stateaware"
+ },
+ "parent": "root",
+ "template": null,
+ "children": [],
+ "absUsedCapacity": {
+ "memory": 1,
+ "vcore": 0
+ },
+ "maxRunningApps": 12,
+ "runningApps": 4,
+ "allocatingAcceptedApps": [
+ "app-1",
+ "app-2"
+ ]
+ }
+ ],
+ "absUsedCapacity": {
+ "memory": 1,
+ "vcore": 0
+ }
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Applications
+
+### Partition applications
+
+Fetch all Applications for the given Partition/State combination and displays general information about the applications like used resources, queue name, submission time and allocations.
+Only following application states are allowed: active, rejected, completed. Active is a fake state that represents all application states except completed and rejected.
+For active state, can narrow the result by status query parameters(case-insensitive). For example, can fetch `Running` applications for the default partition by
+`/ws/v1/partition/default/applications/active?status=running`.
+
+**URL** : `/ws/v1/partition/:partition/applications/:state`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+The content of the application object is the same as Queue Applications. See
+ [Queue Applications](#queue-applications) for details.
+
+### Queue applications
+
+Fetch all Applications for the given Partition/Queue combination and displays general information about the applications like used resources, queue name, submission time and allocations.
+
+**URL** : `/ws/v1/partition/{partitionName}/queue/{queueName}/applications`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Deprecated**:
+
+Field `uuid` has been deprecated, would be removed from below response in YUNIKORN 1.7.0 release. `AllocationID` has replaced `uuid`. Both `uuid` and `AllocationID` fields have the same value. `AllocationID` has extra suffix containing hyphen and counter (-0, -1 and so on) at the end.
+
+**Content examples**
+
+In the example below there are three allocations belonging to two applications, one with a pending request.
+
+```json
+[
+ {
+ "applicationID": "application-0001",
+ "usedResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "maxUsedResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "pendingResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "partition": "default",
+ "queueName": "root.default",
+ "submissionTime": 1648754032076020293,
+ "requests": [
+ {
+ "allocationKey": "f137fab6-3cfa-4536-93f7-bfff92689382",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0001",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task2"
+ },
+ "requestTime": 16487540320812345678,
+ "resource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "pendingCount": 1,
+ "priority": "0",
+ "requiredNodeId": "",
+ "applicationId": "application-0001",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderTimeout": 0,
+ "taskGroupName": "",
+ "allocationLog": [
+ {
+ "message": "node(s) didn't match Pod's node affinity, node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate",
+ "lastOccurrence": 16487540320812346001,
+ "count": 81
+ },
+ {
+ "message": "node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, node(s) didn't match Pod's node affinity",
+ "lastOccurrence": 16487540320812346002,
+ "count": 504
+ },
+ {
+ "message": "node(s) didn't match Pod's node affinity",
+ "lastOccurrence": 16487540320812346003,
+ "count": 1170
+ }
+ ]
+ }
+ ],
+ "allocations": [
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0001",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task0"
+ },
+ "requestTime": 1648754034098912461,
+ "allocationTime": 1648754035973982920,
+ "allocationDelay": 1875070459,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "allocationID": "9af35d44-2d6f-40d1-b51d-758859e6b8a8-0",
+ "resource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "priority": "0",
+ "nodeId": "node-0001",
+ "applicationId": "application-0001",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderUsed": true
+ }
+ ],
+ "applicationState": "Running",
+ "user": "system:serviceaccount:kube-system:deployment-controller",
+ "groups": [
+ "system:serviceaccounts",
+ "system:serviceaccounts:kube-system",
+ "system:authenticated"
+ ],
+ "rejectedMessage": "",
+ "stateLog": [
+ {
+ "time": 1648741409145224000,
+ "applicationState": "Accepted"
+ },
+ {
+ "time": 1648741409145509400,
+ "applicationState": "Starting"
+ },
+ {
+ "time": 1648741409147432100,
+ "applicationState": "Running"
+ }
+ ],
+ "placeholderData": [
+ {
+ "taskGroupName": "task-group-example",
+ "count": 2,
+ "minResource": {
+ "memory": 1000000000,
+ "vcore": 100
+ },
+ "replaced": 1,
+ "timedout": 1
+ }
+ ],
+ "hasReserved": false,
+ "reservations": []
+ },
+ {
+ "applicationID": "application-0002",
+ "usedResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "maxUsedResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "pendingResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "partition": "default",
+ "queueName": "root.default",
+ "submissionTime": 1648754032076020293,
+ "requests": [],
+ "allocations": [
+ {
+ "allocationKey": "54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0002",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task0"
+ },
+ "requestTime": 1648754034098912461,
+ "allocationTime": 1648754035973982920,
+ "allocationDelay": 1875070459,
+ "uuid": "08033f9a-4699-403c-9204-6333856b41bd",
+ "allocationID": "08033f9a-4699-403c-9204-6333856b41bd-0",
+ "resource": {
+ "memory": 2000000000,
+ "vcore": 2000
+ },
+ "priority": "0",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderUsed": false
+ },
+ {
+ "allocationKey": "af3bd2f3-31c5-42dd-8f3f-c2298ebdec81",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0002",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task1"
+ },
+ "requestTime": 1648754034098912461,
+ "allocationTime": 1648754035973982920,
+ "allocationDelay": 1875070459,
+ "uuid": "96beeb45-5ed2-4c19-9a83-2ac807637b3b",
+ "allocationID": "96beeb45-5ed2-4c19-9a83-2ac807637b3b-0",
+ "resource": {
+ "memory": 2000000000,
+ "vcore": 2000
+ },
+ "priority": "0",
+ "nodeId": "node-0002",
+ "applicationId": "application-0002",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderUsed": false
+ }
+ ],
+ "applicationState": "Running",
+ "user": "system:serviceaccount:kube-system:deployment-controller",
+ "groups": [
+ "system:serviceaccounts",
+ "system:serviceaccounts:kube-system",
+ "system:authenticated"
+ ],
+ "rejectedMessage": "",
+ "stateLog": [
+ {
+ "time": 1648741409145224000,
+ "applicationState": "Accepted"
+ },
+ {
+ "time": 1648741409145509400,
+ "applicationState": "Starting"
+ },
+ {
+ "time": 1648741409147432100,
+ "applicationState": "Running"
+ }
+ ],
+ "placeholderData": [],
+ "hasReserved": false,
+ "reservations": []
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Application
+
+### Partition/Queue application
+
+Fetch an Application given a Partition, Queue(optional) and Application ID and displays general information about the application like used resources, queue name, submission time and allocations.
+
+**URL** : `/ws/v1/partition/{partitionName}/application/{appId}` or `/ws/v1/partition/{partitionName}/queue/{queueName}/application/{appId}`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Deprecated**:
+
+Field `uuid` has been deprecated, would be removed from below response in YUNIKORN 1.7.0 release. `AllocationID` has replaced `uuid`. Both `uuid` and `AllocationID` fields have the same value. `AllocationID` has extra suffix containing hyphen and counter (-0, -1 and so on) at the end.
+
+**Content example**
+
+```json
+{
+ "applicationID": "application-0001",
+ "usedResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "maxUsedResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "pendingResource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "partition": "default",
+ "queueName": "root.default",
+ "submissionTime": 1648754032076020293,
+ "requests": [
+ {
+ "allocationKey": "f137fab6-3cfa-4536-93f7-bfff92689382",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0001",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task2"
+ },
+ "requestTime": 16487540320812345678,
+ "resource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "pendingCount": 1,
+ "priority": "0",
+ "requiredNodeId": "",
+ "applicationId": "application-0001",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderTimeout": 0,
+ "taskGroupName": "",
+ "allocationLog": [
+ {
+ "message": "node(s) didn't match Pod's node affinity, node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate",
+ "lastOccurrence": 16487540320812346001,
+ "count": 81
+ },
+ {
+ "message": "node(s) had taint {node-role.kubernetes.io/master: }, that the pod didn't tolerate, node(s) didn't match Pod's node affinity",
+ "lastOccurrence": 16487540320812346002,
+ "count": 504
+ },
+ {
+ "message": "node(s) didn't match Pod's node affinity",
+ "lastOccurrence": 16487540320812346003,
+ "count": 1170
+ }
+ ]
+ }
+ ],
+ "allocations": [
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0001",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task0"
+ },
+ "requestTime": 1648754034098912461,
+ "allocationTime": 1648754035973982920,
+ "allocationDelay": 1875070459,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "allocationID": "9af35d44-2d6f-40d1-b51d-758859e6b8a8-0",
+ "resource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "priority": "0",
+ "nodeId": "node-0001",
+ "applicationId": "application-0001",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderUsed": true
+ }
+ ],
+ "applicationState": "Running",
+ "user": "system:serviceaccount:kube-system:deployment-controller",
+ "groups": [
+ "system:serviceaccounts",
+ "system:serviceaccounts:kube-system",
+ "system:authenticated"
+ ],
+ "rejectedMessage": "",
+ "stateLog": [
+ {
+ "time": 1648741409145224000,
+ "applicationState": "Accepted"
+ },
+ {
+ "time": 1648741409145509400,
+ "applicationState": "Starting"
+ },
+ {
+ "time": 1648741409147432100,
+ "applicationState": "Running"
+ }
+ ],
+ "placeholderData": [
+ {
+ "taskGroupName": "task-group-example",
+ "count": 2,
+ "minResource": {
+ "memory": 1000000000,
+ "vcore": 100
+ },
+ "replaced": 1,
+ "timedout": 1
+ }
+ ],
+ "hasReserved": false,
+ "reservations": []
+}
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## UsersTracker
+### Get users usage tracking information
+
+Fetch all users usage given a Partition and displays general information about the users managed by YuniKorn.
+
+**URL** : `/ws/v1/partition/{partitionName}/usage/users`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content example**
+
+```json
+[
+ {
+ "userName": "user1",
+ "groups": {
+ "app2": "tester"
+ },
+ "queues":
+ {
+ "queuePath": "root",
+ "resourceUsage": {
+ "memory": 12000000000,
+ "vcore": 12000
+ },
+ "runningApplications": ["app1", "app2"],
+ "children": [
+ {
+ "queuePath": "root.default",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app1"],
+ "children": []
+ },
+ {
+ "queuePath": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": [
+ "app2"
+ ],
+ "children": []
+ }]
+ }
+ },
+ {
+ "userName": "user2",
+ "groups": {
+ "app1": "tester"
+ },
+ "queues":
+ {
+ "queuePath": "root",
+ "resourceUsage": {
+ "memory": 11000000000,
+ "vcore": 10000
+ },
+ "runningApplications": ["app1", "app2", "app3"],
+ "children": [
+ {
+ "queuePath": "root.default",
+ "resourceUsage": {
+ "memory": 5000000000,
+ "vcore": 5000
+ },
+ "runningApplications": ["app1"],
+ "children": []
+ },
+ {
+ "queuePath": "root.test",
+ "resourceUsage": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "runningApplications": [
+ "app3"
+ ],
+ "children": []
+ }]
+ }
+ }
+]
+```
+
+### Error response
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## UserTracker
+### Get specific user usage tracking information
+Fetch specific user usage given a Partition and displays general information about the users managed by YuniKorn.
+
+**URL** : `/ws/v1/partition/{partitionName}/usage/user/{userName}`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content example**
+
+```json
+{
+ "userName": "user1",
+ "groups": {
+ "app1": "tester"
+ },
+ "queues":
+ {
+ "queuePath": "root",
+ "resourceUsage": {
+ "memory": 12000000000,
+ "vcore": 12000
+ },
+ "runningApplications": ["app1", "app2"],
+ "children": [
+ {
+ "queuePath": "root.default",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app1"],
+ "children": []
+ },
+ {
+ "queuePath": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": [
+ "app2"
+ ],
+ "children": []
+ }]
+ }
+}
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## GroupsTracker
+### Get groups usage tracking information
+Fetch all groups usage given a Partition and displays general information about the groups managed by YuniKorn.
+
+**URL** : `/ws/v1/partition/{partitionName}/usage/groups`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content example**
+
+```json
+[
+ {
+ "groupName": "group1",
+ "applications": ["app1", "app2"],
+ "queues":
+ {
+ "queuePath": "root",
+ "resourceUsage": {
+ "memory": 12000000000,
+ "vcore": 12000
+ },
+ "runningApplications": ["app1", "app2"],
+ "children": [
+ {
+ "queuePath": "root.default",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app1"],
+ "children": []
+ },
+ {
+ "queuePath": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": [
+ "app2"
+ ],
+ "children": []
+ }]
+ }
+ },
+ {
+ "groupName": "group2",
+ "applications": ["app1", "app2", "app3"],
+ "queues":
+ {
+ "queuePath": "root",
+ "resourceUsage": {
+ "memory": 11000000000,
+ "vcore": 10000
+ },
+ "runningApplications": ["app1", "app2", "app3"],
+ "children": [
+ {
+ "queuePath": "root.default",
+ "resourceUsage": {
+ "memory": 5000000000,
+ "vcore": 5000
+ },
+ "runningApplications": ["app1"],
+ "children": []
+ },
+ {
+ "queuePath": "root.test",
+ "resourceUsage": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "runningApplications": [
+ "app3"
+ ],
+ "children": []
+ }]
+ }
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## GroupTracker
+### Get specific group usage tracking information
+
+Fetch specific group usage given a Partition and displays general information about the groups managed by YuniKorn.
+
+**URL** : `/ws/v1/partition/{partitionName}/usage/group/{groupName}`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content example**
+
+```json
+{
+ "groupName": "group1",
+ "applications": ["app1", "app2"],
+ "queues":
+ {
+ "queuePath": "root",
+ "resourceUsage": {
+ "memory": 12000000000,
+ "vcore": 12000
+ },
+ "runningApplications": ["app1", "app2"],
+ "children": [
+ {
+ "queuePath": "root.default",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app1"],
+ "children": []
+ },
+ {
+ "queuePath": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": [
+ "app2"
+ ],
+ "children": []
+ }]
+ }
+}
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Nodes
+
+### Partition nodes
+
+Fetch all Nodes associated with given Partition and displays general information about the nodes managed by YuniKorn.
+Node details include host and rack name, capacity, resources, utilization, and allocations.
+
+**URL** : `/ws/v1/partition/{partitionName}/nodes`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+Here you can see an example response from a 2-node cluster having 3 allocations.
+
+```json
+[
+ {
+ "nodeID": "node-0001",
+ "hostName": "",
+ "rackName": "",
+ "attributes": {
+ "beta.kubernetes.io/arch": "amd64",
+ "beta.kubernetes.io/os": "linux",
+ "kubernetes.io/arch": "amd64",
+ "kubernetes.io/hostname": "node-0001",
+ "kubernetes.io/os": "linux",
+ "node-role.kubernetes.io/control-plane": "",
+ "node-role.kubernetes.io/master": "",
+ "node.kubernetes.io/exclude-from-external-load-balancers": "",
+ "ready": "true",
+ "si.io/hostname": "node-0001",
+ "si.io/rackname": "/rack-default",
+ "si/instance-type": "",
+ "si/node-partition": "[mycluster]default"
+ },
+ "capacity": {
+ "ephemeral-storage": 75850798569,
+ "hugepages-1Gi": 0,
+ "hugepages-2Mi": 0,
+ "memory": 14577000000,
+ "pods": 110,
+ "vcore": 10000
+ },
+ "allocated": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "occupied": {
+ "memory": 154000000,
+ "vcore" :750
+ },
+ "available": {
+ "ephemeral-storage": 75850798569,
+ "hugepages-1Gi": 0,
+ "hugepages-2Mi": 0,
+ "memory": 6423000000,
+ "pods": 110,
+ "vcore": 1250
+ },
+ "utilized": {
+ "memory": 3,
+ "vcore": 13
+ },
+ "allocations": [
+ {
+ "allocationKey": "54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0001",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task0"
+ },
+ "requestTime": 1648754034098912461,
+ "allocationTime": 1648754035973982920,
+ "allocationDelay": 1875070459,
+ "uuid": "08033f9a-4699-403c-9204-6333856b41bd",
+ "allocationID": "08033f9a-4699-403c-9204-6333856b41bd-0",
+ "resource": {
+ "memory": 2000000000,
+ "vcore": 2000
+ },
+ "priority": "0",
+ "nodeId": "node-0001",
+ "applicationId": "application-0001",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderUsed": false
+ },
+ {
+ "allocationKey": "deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0002",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task0"
+ },
+ "requestTime": 1648754034098912461,
+ "allocationTime": 1648754035973982920,
+ "allocationDelay": 1875070459,
+ "uuid": "9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "allocationID": "9af35d44-2d6f-40d1-b51d-758859e6b8a8-0",
+ "resource": {
+ "memory": 4000000000,
+ "vcore": 4000
+ },
+ "priority": "0",
+ "nodeId": "node-0001",
+ "applicationId": "application-0002",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderUsed": false
+ }
+ ],
+ "schedulable": true
+ },
+ {
+ "nodeID": "node-0002",
+ "hostName": "",
+ "rackName": "",
+ "attributes": {
+ "beta.kubernetes.io/arch": "amd64",
+ "beta.kubernetes.io/os": "linux",
+ "kubernetes.io/arch": "amd64",
+ "kubernetes.io/hostname": "node-0002",
+ "kubernetes.io/os": "linux",
+ "ready": "false",
+ "si.io/hostname": "node-0002",
+ "si.io/rackname": "/rack-default",
+ "si/instance-type": "",
+ "si/node-partition": "[mycluster]default"
+ },
+ "capacity": {
+ "ephemeral-storage": 75850798569,
+ "hugepages-1Gi": 0,
+ "hugepages-2Mi": 0,
+ "memory": 14577000000,
+ "pods": 110,
+ "vcore": 10000
+ },
+ "allocated": {
+ "memory": 2000000000,
+ "vcore": 2000
+ },
+ "occupied": {
+ "memory": 154000000,
+ "vcore" :750
+ },
+ "available": {
+ "ephemeral-storage": 75850798569,
+ "hugepages-1Gi": 0,
+ "hugepages-2Mi": 0,
+ "memory": 6423000000,
+ "pods": 110,
+ "vcore": 1250
+ },
+ "utilized": {
+ "memory": 8,
+ "vcore": 38
+ },
+ "allocations": [
+ {
+ "allocationKey": "af3bd2f3-31c5-42dd-8f3f-c2298ebdec81",
+ "allocationTags": {
+ "kubernetes.io/label/app": "sleep",
+ "kubernetes.io/label/applicationId": "application-0001",
+ "kubernetes.io/label/queue": "root.default",
+ "kubernetes.io/meta/namespace": "default",
+ "kubernetes.io/meta/podName": "task1"
+ },
+ "requestTime": 1648754034098912461,
+ "allocationTime": 1648754035973982920,
+ "allocationDelay": 1875070459,
+ "uuid": "96beeb45-5ed2-4c19-9a83-2ac807637b3b",
+ "allocationID": "96beeb45-5ed2-4c19-9a83-2ac807637b3b-0",
+ "resource": {
+ "memory": 2000000000,
+ "vcore": 2000
+ },
+ "priority": "0",
+ "nodeId": "node-0002",
+ "applicationId": "application-0001",
+ "partition": "default",
+ "placeholder": false,
+ "placeholderUsed": false
+ }
+ ],
+ "schedulable": true,
+ "isReserved": false,
+ "reservations": []
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Node
+
+### Partition node
+
+Fetch a Node associated with given Partition and Node ID and displays general information about the node managed by YuniKorn.
+Node details include host and rack name, capacity, resources, utilization, and allocations.
+
+**URL** : `/ws/v1/partition/{partitionName}/node/{nodeId}`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+{
+ "nodeID":"node-0001",
+ "hostName":"",
+ "rackName":"",
+ "capacity":{
+ "ephemeral-storage":75850798569,
+ "hugepages-1Gi":0,
+ "hugepages-2Mi":0,
+ "memory":14577000000,
+ "pods":110,
+ "vcore":10000
+ },
+ "allocated":{
+ "memory":6000000000,
+ "vcore":6000
+ },
+ "occupied":{
+ "memory":154000000,
+ "vcore":750
+ },
+ "available":{
+ "ephemeral-storage":75850798569,
+ "hugepages-1Gi":0,
+ "hugepages-2Mi":0,
+ "memory":6423000000,
+ "pods":110,
+ "vcore":1250
+ },
+ "utilized":{
+ "memory":3,
+ "vcore":13
+ },
+ "allocations":[
+ {
+ "allocationKey":"54e5d77b-f4c3-4607-8038-03c9499dd99d",
+ "allocationTags":{
+ "kubernetes.io/label/app":"sleep",
+ "kubernetes.io/label/applicationId":"application-0001",
+ "kubernetes.io/label/queue":"root.default",
+ "kubernetes.io/meta/namespace":"default",
+ "kubernetes.io/meta/podName":"task0"
+ },
+ "requestTime":1648754034098912461,
+ "allocationTime":1648754035973982920,
+ "allocationDelay":1875070459,
+ "uuid":"08033f9a-4699-403c-9204-6333856b41bd",
+ "allocationID":"08033f9a-4699-403c-9204-6333856b41bd-0",
+ "resource":{
+ "memory":2000000000,
+ "vcore":2000
+ },
+ "priority":"0",
+ "nodeId":"node-0001",
+ "applicationId":"application-0001",
+ "partition":"default",
+ "placeholder":false,
+ "placeholderUsed":false
+ },
+ {
+ "allocationKey":"deb12221-6b56-4fe9-87db-ebfadce9aa20",
+ "allocationTags":{
+ "kubernetes.io/label/app":"sleep",
+ "kubernetes.io/label/applicationId":"application-0002",
+ "kubernetes.io/label/queue":"root.default",
+ "kubernetes.io/meta/namespace":"default",
+ "kubernetes.io/meta/podName":"task0"
+ },
+ "requestTime":1648754034098912461,
+ "allocationTime":1648754035973982920,
+ "allocationDelay":1875070459,
+ "uuid":"9af35d44-2d6f-40d1-b51d-758859e6b8a8",
+ "allocationID":"9af35d44-2d6f-40d1-b51d-758859e6b8a8-0",
+ "resource":{
+ "memory":4000000000,
+ "vcore":4000
+ },
+ "priority":"0",
+ "nodeId":"node-0001",
+ "applicationId":"application-0002",
+ "partition":"default",
+ "placeholder":false,
+ "placeholderUsed":false
+ }
+ ],
+ "schedulable":true
+}
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Node utilization
+
+Show how every node is distributed with regard to dominant resource utilization.
+
+**Status** : Deprecated since v1.5.0 and will be removed in the next major release. Replaced with `/ws/v1/scheduler/node-utilizations`.
+
+**URL** : `/ws/v1/scheduler/node-utilization`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+{
+ "type": "vcore",
+ "utilization": [
+ {
+ "bucketName": "0-10%",
+ "numOfNodes": 1,
+ "nodeNames": [
+ "aethergpu"
+ ]
+ },
+ {
+ "bucketName": "10-20%",
+ "numOfNodes": 2,
+ "nodeNames": [
+ "primary-node",
+ "second-node"
+ ]
+ },
+ ...
+ ]
+}
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Node utilizations
+
+Show the nodes utilization of different types of resources in a cluster.
+
+**URL** : `/ws/v1/scheduler/node-utilizations`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "clusterId": "mycluster",
+ "partition": "default",
+ "utilizations": [
+ {
+ "type": "pods",
+ "utilization": [
+ {
+ "bucketName": "0-10%",
+ "numOfNodes": 2,
+ "nodeNames": [
+ "primary-node",
+ "second-node"
+ ]
+ },
+ {
+ "bucketName": "10-20%"
+ },
+ ...
+ ]
+ },
+ {
+ "type": "vcores",
+ "utilization": [
+ {
+ "bucketName": "0-10%",
+ "numOfNodes": 2,
+ "nodeNames": [
+ "primary-node",
+ "second-node"
+ ]
+ },
+ {
+ "bucketName": "10-20%"
+ },
+ ...
+ ]
+ },
+ ...
+ ]
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Goroutines info
+
+Dumps the stack traces of the currently running goroutines.
+
+**URL** : `/ws/v1/stack`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+goroutine 356 [running
+]:
+github.com/apache/yunikorn-core/pkg/webservice.getStackInfo.func1(0x30a0060,
+0xc003e900e0,
+0x2)
+ /yunikorn/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/handlers.go: 41 +0xab
+github.com/apache/yunikorn-core/pkg/webservice.getStackInfo(0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /yunikorn/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/handlers.go: 48 +0x71
+net/http.HandlerFunc.ServeHTTP(0x2df0e10,
+0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /usr/local/go/src/net/http/server.go: 1995 +0x52
+github.com/apache/yunikorn-core/pkg/webservice.Logger.func1(0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /yunikorn/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20200717041747-f3e1c760c714/pkg/webservice/webservice.go: 65 +0xd4
+net/http.HandlerFunc.ServeHTTP(0xc00003a570,
+0x30a0060,
+0xc003e900e0,
+0xc00029ba00)
+ /usr/local/go/src/net/http/server.go: 1995 +0x52
+github.com/gorilla/mux.(*Router).ServeHTTP(0xc00029cb40,
+0x30a0060,
+0xc003e900e0,
+0xc0063fee00)
+ /yunikorn/go/pkg/mod/github.com/gorilla/mux@v1.7.3/mux.go: 212 +0x140
+net/http.serverHandler.ServeHTTP(0xc0000df520,
+0x30a0060,
+0xc003e900e0,
+0xc0063fee00)
+ /usr/local/go/src/net/http/server.go: 2774 +0xcf
+net/http.(*conn).serve(0xc0000eab40,
+0x30a61a0,
+0xc003b74000)
+ /usr/local/go/src/net/http/server.go: 1878 +0x812
+created by net/http.(*Server).Serve
+ /usr/local/go/src/net/http/server.go: 2884 +0x4c5
+
+goroutine 1 [chan receive,
+ 26 minutes
+]:
+main.main()
+ /yunikorn/pkg/shim/main.go: 52 +0x67a
+
+goroutine 19 [syscall,
+ 26 minutes
+]:
+os/signal.signal_recv(0x1096f91)
+ /usr/local/go/src/runtime/sigqueue.go: 139 +0x9f
+os/signal.loop()
+ /usr/local/go/src/os/signal/signal_unix.go: 23 +0x30
+created by os/signal.init.0
+ /usr/local/go/src/os/signal/signal_unix.go: 29 +0x4f
+
+...
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Metrics
+
+Endpoint to retrieve metrics from the Prometheus server.
+The metrics are dumped with help messages and type information.
+
+**URL** : `/ws/v1/metrics`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
+# TYPE go_gc_duration_seconds summary
+go_gc_duration_seconds{quantile="0"} 2.567e-05
+go_gc_duration_seconds{quantile="0.25"} 3.5727e-05
+go_gc_duration_seconds{quantile="0.5"} 4.5144e-05
+go_gc_duration_seconds{quantile="0.75"} 6.0024e-05
+go_gc_duration_seconds{quantile="1"} 0.00022528
+go_gc_duration_seconds_sum 0.021561648
+go_gc_duration_seconds_count 436
+# HELP go_goroutines Number of goroutines that currently exist.
+# TYPE go_goroutines gauge
+go_goroutines 82
+# HELP go_info Information about the Go environment.
+# TYPE go_info gauge
+go_info{version="go1.12.17"} 1
+# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
+# TYPE go_memstats_alloc_bytes gauge
+go_memstats_alloc_bytes 9.6866248e+07
+
+...
+
+# HELP yunikorn_scheduler_vcore_nodes_usage Nodes resource usage, by resource name.
+# TYPE yunikorn_scheduler_vcore_nodes_usage gauge
+yunikorn_scheduler_vcore_nodes_usage{range="(10%, 20%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(20%,30%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(30%,40%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(40%,50%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(50%,60%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(60%,70%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(70%,80%]"} 1
+yunikorn_scheduler_vcore_nodes_usage{range="(80%,90%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="(90%,100%]"} 0
+yunikorn_scheduler_vcore_nodes_usage{range="[0,10%]"} 0
+```
+
+## Configuration validation
+
+**URL** : `/ws/v1/validate-conf`
+
+**Method** : `POST`
+
+**Auth required** : NO
+
+### Success response
+
+Regardless whether the configuration is allowed or not if the server was able to process the request, it will yield a 200 HTTP status code.
+
+**Code** : `200 OK`
+
+#### Allowed configuration
+
+Sending the following simple configuration yields an accept
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: test
+```
+
+Reponse
+
+```json
+{
+ "allowed": true,
+ "reason": ""
+}
+```
+
+#### Disallowed configuration
+
+The following configuration is not allowed due to the "wrong_text" field put into the yaml file.
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: test
+ - wrong_text
+```
+
+Reponse
+
+```json
+{
+ "allowed": false,
+ "reason": "yaml: unmarshal errors:\n line 7: cannot unmarshal !!str `wrong_text` into configs.PartitionConfig"
+}
+```
+
+## Configuration
+
+Endpoint to retrieve the current scheduler configuration
+
+**URL** : `/ws/v1/config`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content example (with `Accept: application/json` header)**
+
+```json
+{
+ "Partitions": [
+ {
+ "Name": "default",
+ "Queues": [
+ {
+ "Name": "root",
+ "Parent": true,
+ "Resources": {},
+ "SubmitACL": "*",
+ "ChildTemplate": {
+ "Resources": {}
+ }
+ }
+ ],
+ "PlacementRules": [
+ {
+ "Name": "tag",
+ "Create": true,
+ "Filter": {
+ "Type": ""
+ },
+ "Value": "namespace"
+ }
+ ],
+ "Preemption": {
+ "Enabled": false
+ },
+ "NodeSortPolicy": {
+ "Type": ""
+ }
+ }
+ ],
+ "Checksum": "FD5D3726DF0F02416E02F3919D78F61B15D14425A34142D93B24C137ED056946",
+ "Extra": {
+ "event.trackingEnabled": "false",
+ "log.core.scheduler.level": "info",
+ "log.core.security.level": "info",
+ "log.level": "debug"
+ }
+}
+```
+
+**Content example (without `Accept: application/json` header)**
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ parent: true
+ submitacl: "*"
+ placementrules:
+ - name: tag
+ create: true
+ value: namespace
+checksum: FD5D3726DF0F02416E02F3919D78F61B15D14425A34142D93B24C137ED056946
+extra:
+ event.trackingEnabled: "false"
+ log.core.scheduler.level: info
+ log.core.security.level: info
+ log.level: debug
+
+```
+
+## Application history
+
+Endpoint to retrieve historical data about the number of total applications by timestamp.
+
+**URL** : `/ws/v1/history/apps`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "timestamp": 1595939966153460000,
+ "totalApplications": "1"
+ },
+ {
+ "timestamp": 1595940026152892000,
+ "totalApplications": "1"
+ },
+ {
+ "timestamp": 1595940086153799000,
+ "totalApplications": "2"
+ },
+ {
+ "timestamp": 1595940146154497000,
+ "totalApplications": "2"
+ },
+ {
+ "timestamp": 1595940206155187000,
+ "totalApplications": "2"
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+## Container history
+
+Endpoint to retrieve historical data about the number of total containers by timestamp.
+
+**URL** : `/ws/v1/history/containers`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+[
+ {
+ "timestamp": 1595939966153460000,
+ "totalContainers": "1"
+ },
+ {
+ "timestamp": 1595940026152892000,
+ "totalContainers": "1"
+ },
+ {
+ "timestamp": 1595940086153799000,
+ "totalContainers": "3"
+ },
+ {
+ "timestamp": 1595940146154497000,
+ "totalContainers": "3"
+ },
+ {
+ "timestamp": 1595940206155187000,
+ "totalContainers": "3"
+ }
+]
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+**Content examples**
+
+```json
+{
+ "status_code": 500,
+ "message": "system error message. for example, json: invalid UTF-8 in string: ..",
+ "description": "system error message. for example, json: invalid UTF-8 in string: .."
+}
+```
+
+
+## Endpoint healthcheck
+
+Endpoint to retrieve historical data about critical logs, negative resource on node/cluster/app, ...
+
+**URL** : `/ws/v1/scheduler/healthcheck`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```json
+{
+ "Healthy": true,
+ "HealthChecks": [
+ {
+ "Name": "Scheduling errors",
+ "Succeeded": true,
+ "Description": "Check for scheduling error entries in metrics",
+ "DiagnosisMessage": "There were 0 scheduling errors logged in the metrics"
+ },
+ {
+ "Name": "Failed nodes",
+ "Succeeded": true,
+ "Description": "Check for failed nodes entries in metrics",
+ "DiagnosisMessage": "There were 0 failed nodes logged in the metrics"
+ },
+ {
+ "Name": "Negative resources",
+ "Succeeded": true,
+ "Description": "Check for negative resources in the partitions",
+ "DiagnosisMessage": "Partitions with negative resources: []"
+ },
+ {
+ "Name": "Negative resources",
+ "Succeeded": true,
+ "Description": "Check for negative resources in the nodes",
+ "DiagnosisMessage": "Nodes with negative resources: []"
+ },
+ {
+ "Name": "Consistency of data",
+ "Succeeded": true,
+ "Description": "Check if a node's allocated resource <= total resource of the node",
+ "DiagnosisMessage": "Nodes with inconsistent data: []"
+ },
+ {
+ "Name": "Consistency of data",
+ "Succeeded": true,
+ "Description": "Check if total partition resource == sum of the node resources from the partition",
+ "DiagnosisMessage": "Partitions with inconsistent data: []"
+ },
+ {
+ "Name": "Consistency of data",
+ "Succeeded": true,
+ "Description": "Check if node total resource = allocated resource + occupied resource + available resource",
+ "DiagnosisMessage": "Nodes with inconsistent data: []"
+ },
+ {
+ "Name": "Consistency of data",
+ "Succeeded": true,
+ "Description": "Check if node capacity >= allocated resources on the node",
+ "DiagnosisMessage": "Nodes with inconsistent data: []"
+ },
+ {
+ "Name": "Reservation check",
+ "Succeeded": true,
+ "Description": "Check the reservation nr compared to the number of nodes",
+ "DiagnosisMessage": "Reservation/node nr ratio: [0.000000]"
+ }
+ ]
+}
+```
+
+## Retrieve full state dump
+
+Endpoint to retrieve the following information in a single response:
+
+* Current timestamp (Unix timestamp in nanosecond)
+* List of partitions
+* List of applications (running, completed and rejected)
+* Application history
+* Nodes
+* Generic cluster information
+* Container history
+* Queues
+* RMDiagnostics
+* Log level
+* Configuration
+
+**URL** : `/ws/v1/fullstatedump`
+
+**Method** : `GET`
+
+**Auth required** : NO
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+The output of this REST query can be rather large, and it is a combination of those which have already been demonstrated.
+
+The `RMDiagnostics` shows the content of the K8Shim cache. The exact content is version dependent and has not stabilised.
+The current content shows the cached objects:
+* nodes
+* pods
+* priorityClasses
+* schedulingState (pod status)
+
+### Failure response
+
+**Code**: `500 Internal Server Error`
+
+## Batch Events
+
+Endpoint is used to retrieve a batch of event records.
+
+**URL**: `/ws/v1/events/batch`
+
+**METHOD** : `GET`
+
+**Auth required** : NO
+
+**URL query parameters** :
+- `count` (optional) : Specifies the maxmem number of events to be included in the response.
+- `start` (optional) : Specifies the starting ID for retrieving events. If the specified ID is outside the ring buffer
+(too low or too high), the response will include the lowest and highest ID values with `EventRecords` being empty.
+
+
+### Success response
+
+**Code**: `200 OK`
+
+**Content examples**
+
+```json
+{
+ "InstanceUUID": "400046c6-2180-41a2-9be1-1c251ab2c498",
+ "LowestID": 0,
+ "HighestID": 7,
+ "EventRecords": [
+ {
+ "type": 3,
+ "objectID": "yk8s-worker",
+ "message": "schedulable: true",
+ "timestampNano": 1701347180239597300,
+ "eventChangeType": 1,
+ "eventChangeDetail": 302,
+ "resource": {}
+ },
+ {
+ "type": 3,
+ "objectID": "yk8s-worker",
+ "message": "Node added to the scheduler",
+ "timestampNano": 1701347180239650600,
+ "eventChangeType": 2,
+ "resource": {
+ "resources": {
+ "ephemeral-storage": {
+ "value": 502921060352
+ },
+ "hugepages-1Gi": {},
+ "hugepages-2Mi": {},
+ "memory": {
+ "value": 33424998400
+ },
+ "pods": {
+ "value": 110
+ },
+ "vcore": {
+ "value": 8000
+ }
+ }
+ }
+ }
+ ]
+}
+```
+
+### Error response
+
+**Code** : `500 Internal Server Error`
+
+### Event stream
+
+Creates a persistent HTTP connection for event streaming. New events are sent to the clients immediately, so unlike the batch interface, there is no need for polling.
+The number of active connections is limited. The default setting is 100 connections total and 15 connections per host. The respective configmap properties are `event.maxStreams` and `event.maxStreamsPerHost`.
+
+**URL**: `/ws/v1/events/stream`
+
+**METHOD** : `GET`
+
+**Auth required** : NO
+
+**URL query parameters**:
+- `count` (optional) : Specifies the number of past events (those which have been generated before the connection establishment) to include in the response. Default value is 0.
+
+### Success response
+
+**Code**: `200 OK`
+
+**Content examples**
+
+```json
+{"type":2,"objectID":"app-1","timestampNano":1708465452903045265,"eventChangeType":1,"eventChangeDetail":204,"resource":{}}
+{"type":2,"objectID":"app-1","timestampNano":1708465452903192898,"eventChangeType":2,"eventChangeDetail":201,"referenceID":"alloc-1","resource":{"resources":{"memory":{"value":10000000},"vcore":{"value":1000}}}}
+{"type":3,"objectID":"node-1:1234","timestampNano":1708465452903312146,"eventChangeType":2,"eventChangeDetail":303,"referenceID":"alloc-1","resource":{"resources":{"memory":{"value":10000000},"vcore":{"value":1000}}}}
+{"type":2,"objectID":"app-1","timestampNano":1708465452903474210,"eventChangeType":1,"eventChangeDetail":205,"resource":{}}
+{"type":5,"objectID":"testuser","timestampNano":1708465452903506166,"eventChangeType":2,"eventChangeDetail":603,"referenceID":"root.singleleaf","resource":{"resources":{"memory":{"value":10000000},"vcore":{"value":1000}}}}
+```
+
+### Error responses
+
+**Code** : `400 Bad Request` (URL query is invalid)
+**Code** : `503 Service Unavailable` (Too many active streaming connections)
+**Code** : `500 Internal Server Error`
diff --git a/versioned_docs/version-1.5.0/api/system.md b/versioned_docs/version-1.5.0/api/system.md
new file mode 100644
index 0000000..1d685ff
--- /dev/null
+++ b/versioned_docs/version-1.5.0/api/system.md
@@ -0,0 +1,225 @@
+---
+id: system
+title: System
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+These endpoints are for the [pprof](https://github.com/google/pprof) profiling tool.
+
+## pprof
+
+**URL** : `/debug/pprof/`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```text
+/debug/pprof/
+
+Types of profiles available:
+Count Profile
+273 allocs
+0 block
+0 cmdline
+78 goroutine
+273 heap
+0 mutex
+0 profile
+29 threadcreate
+0 trace
+full goroutine stack dump
+Profile Descriptions:
+
+allocs: A sampling of all past memory allocations
+block: Stack traces that led to blocking on synchronization primitives
+cmdline: The command line invocation of the current program
+goroutine: Stack traces of all current goroutines
+heap: A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.
+mutex: Stack traces of holders of contended mutexes
+profile: CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.
+threadcreate: Stack traces that led to the creation of new OS threads
+trace: A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.
+```
+
+## Heap
+
+**URL** : `/debug/pprof/heap`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Thread create
+
+**URL** : `/debug/pprof/threadcreate`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Goroutine
+
+**URL** : `/debug/pprof/goroutine`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Allocations
+
+**URL** : `/debug/pprof/allocs`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Block
+
+**URL** : `/debug/pprof/block`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Mutex
+
+**URL** : `/debug/pprof/mutex`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Cmdline
+
+**URL** : `/debug/pprof/cmdline`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Profile
+
+**URL** : `/debug/pprof/profile`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Symbol
+
+**URL** : `/debug/pprof/symbol`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
+
+## Trace
+
+**URL** : `/debug/pprof/trace`
+
+**Method** : `GET`
+
+### Success response
+
+**Code** : `200 OK`
+
+**Content examples**
+
+```proto
+// binary data from proto
+```
diff --git a/versioned_docs/version-1.5.0/archived_design/cross_queue_preemption.md b/versioned_docs/version-1.5.0/archived_design/cross_queue_preemption.md
new file mode 100644
index 0000000..51c8033
--- /dev/null
+++ b/versioned_docs/version-1.5.0/archived_design/cross_queue_preemption.md
@@ -0,0 +1,126 @@
+---
+id: cross_queue_preemption
+title: Cross Queue Preemption
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Problems:
+
+According to lessons we learned from YARN Scheduler preemption.
+
+**Here're top bad things:**
+
+- Preemption is a shotgun instead of a sniper, when a preemption decision is made, nobody knows if preempted resources will go to demanding queue/app/user or not.
+- Preemption logic and allocation is different, we have to implement (and mimic) what we have done in scheduler allocation logic.
+
+**Here're top good things:**
+
+- Preemption is fast (thanks to the shotgun), reclaiming thousands of containers only takes ~ 1 sec.
+- We have understand how painful it is to handle DRF, multiple preemption policies (inter/intra-queue, shotgun/surgical preemption, etc.) And we have developed some good logic
+to make sure better modularization and plug-ability
+
+## Answer some questions for design/implementation choices
+
+**1\. Do we really want preemption-delay? (Or we just want to control pace)**
+
+In CS, we have preemption-delay, which select victims in preemption candidates, wait for a certain time before killing it.
+
+The purposes of preemption delay are: a. give heads-up time to apps so
+they can prepare bad things happen (unfortunately no app do anything for these heads up, at least from what I knew). b. control preemption pace.
+
+And in practice, I found it causes a lot of issues, for example when a
+cluster state keep changing, it is very hard to ensure accurate preemption.
+
+**Proposal:**
+
+Remove the preemption-delay, keep the logics of controlling preemption pace. (such as ```yarn.resourcemanager.monitor.capacity.preemption
+.total_preemption_per_round```). And we can do allocation together with preemption.
+This don't mean containers will be stopped immediately after preemption issued. Instead, RM can control delays between signal a container and kill a container. Such as grace
+termination of POD in K8s: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
+
+**2\. Do we want to do preemption for every scheduling logic, or we can do periodically?**
+
+In CS, we have preemption logic runs periodically, like every 1 sec or 3 sec.
+
+Since preemption logic involves some heavy logics, like calculating shares of queues/apps. And when doing accurate preemption, we may need to scan nodes for preemption candidate.
+Considering this, I propose to have preemption runs periodically. But it is important to note that, we need to try to use as much code as possible for
+allocation-inside-preemption, otherwise there will be too much duplicated logic and very hard to be maintained in the future.
+
+**3\. Preemption cost and function**
+
+We found it is helpful to add cost for preemption, such as container live time, priority, type of container. It could be a cost function (Which returns a numeric value) or it
+could be a comparator (which compare two allocations for preemption ask).
+
+## Pseudo code
+
+Logic of allocation (invoked every allocation cycle)
+
+```
+input:
+ - nAlloc, allocate N allocations for this allocation cycle.
+
+for partition:
+ askCandidates := findAskCandidates(nAlloc, preemption=false)
+
+ allocated, failed_to_allocated := tryAllocate(askCandidates);
+
+ send-allocated-to-cache-to-commit;
+
+ update-missed-opportunity (allocated, failed_to_allocated);
+
+ nAlloc -= len(allocated)
+```
+
+Logic of preemption (invoked every preemption cycle)
+
+```
+// It has to be done for every preemption-policy because calculation is different.
+for preemption-policy:
+ preempt_results := policy.preempt()
+ for preempt_results:
+ send-preempt-result-to-cache-to-commit;
+ updated-missed-opportunity (allocated)
+```
+
+Inside preemption policy
+
+```
+inter-queue-preempt-policy:
+ calculate-preemption-quotas;
+
+ for partitions:
+ total_preempted := resource(0);
+
+ while total_preempted < partition-limited:
+ // queues will be sorted by allocating - preempting
+ // And ignore any key in preemption_mask
+ askCandidates := findAskCandidates(N, preemption=true)
+
+ preempt_results := tryAllocate(askCandidates, preemption=true);
+
+ total_preempted += sigma(preempt_result.allocResource)
+
+ send-allocated-to-cache-to-commit;
+
+ update-missed-opportunity (allocated, failed_to_allocated);
+
+ update-preemption-mask(askCandidates.allocKeys - preempt_results.allocKeys)
+```
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/archived_design/k8shim.md b/versioned_docs/version-1.5.0/archived_design/k8shim.md
new file mode 100644
index 0000000..01f08db
--- /dev/null
+++ b/versioned_docs/version-1.5.0/archived_design/k8shim.md
@@ -0,0 +1,74 @@
+---
+id: k8shim
+title: Kubernetes Shim Design
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Github repo: https://github.com/apache/yunikorn-k8shim
+
+Please read the [architecture](../design/architecture.md) doc before reading this one, you will need to understand
+the 3 layer design of YuniKorn before getting to understand what is the Kubernetes shim.
+
+## The Kubernetes shim
+
+The YuniKorn Kubernetes shim is responsible for talking to Kubernetes, it is responsible for translating the Kubernetes
+cluster resources, and resource requests via scheduler interface and send them to the scheduler core.
+And when a scheduler decision is made, it is responsible for binding the pod to the specific node. All the communication
+between the shim and the scheduler core is through the scheduler-interface.
+
+## The admission controller
+
+The admission controller runs in a separate pod, it runs a
+[mutation webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#mutatingadmissionwebhook)
+and a [validation webhook](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook), where:
+
+1. The `mutation webhook` mutates pod spec by:
+ - Adding `schedulerName: yunikorn`
+ - By explicitly specifying the scheduler name, the pod will be scheduled by YuniKorn scheduler.
+ - Adding `applicationId` label
+ - When a label `applicationId` exists, reuse the given applicationId.
+ - When a label `spark-app-selector` exists, reuse the given spark app ID.
+ - Otherwise, assign a generated application ID for this pod, using convention: `yunikorn-<namespace>-autogen`. This is unique per namespace.
+ - Adding `queue` label
+ - When a label `queue` exists, reuse the given queue name. Note, if placement rule is enabled, values set in the label is ignored.
+ - Otherwise, adds `queue: root.default`
+ - Adding `disableStateAware` label
+ - If pod was assigned a generated applicationId by the admission controller, also set `disableStateAware: true`. This causes the generated application
+ to immediately transition from the `Starting` to `Running` state so that it will not block other applications.
+2. The `validation webhook` validates the configuration set in the configmap
+ - This is used to prevent writing malformed configuration into the configmap.
+ - The validation webhook calls scheduler [validation REST API](api/scheduler.md#configuration-validation) to validate configmap updates.
+
+### Admission controller deployment
+
+By default, the admission controller is deployed as part of the YuniKorn Helm chart installation. This can be disabled if necessary (though not recommended) by setting the Helm parameter `embedAdmissionController` to `false`.
+
+On startup, the admission controller performs a series of tasks to ensure that it is properly registered with Kubernetes:
+1. Loads a Kubernetes secret called `admission-controller-secrets`. This secret stores a pair of CA certificates which are used to sign the TLS server certificate used by the admission controller.
+2. If the secret cannot be found or either CA certificate is within 90 days of expiration, generates new certificate(s). If a certificate is expiring, a new one is generated with an expiration of 12 months in the future. If both certificates are missing or expiring, the second certificate is generated with an expiration of 6 months in the future. This ensures that both certificates do not expire at the same time, and that there is an overlap of trusted certificates.
+3. If the CA certificates were created or updated, writes the secrets back to Kubernetes.
+4. Generates an ephemeral TLS server certificate signed by the CA certificate with the latest expiration date.
+5. Validates, and if necessary, creates or updates the Kubernetes webhook configurations named `yunikorn-admission-controller-validations` and `yunikorn-admission-controller-mutations`. If the CA certificates have changed, the webhooks will also be updated. These webhooks allow the Kubernetes API server to connect to the admission controller service to perform configmap validations and pod mutations.
+6. Starts up the admission controller HTTPS server.
+
+Additionally, the admission controller also starts a background task to wait for CA certificates to expire. Once either certificate is expiring within the next 30 days, new CA and server certificates are generated, the webhook configurations are updated, and the HTTPS server is quickly restarted. This ensures that certificates rotate properly without downtime.
+
+In production clusters, it is recommended to deploy the admission controller with two replicas by setting the Helm parameter `admissionController.replicaCount` to `2`. This will ensure that at least one admission controller webhook is reachable by the Kubernetes API server at all times. In this configuration, the CA certificates and webhook configurations are shared between the instances.
diff --git a/versioned_docs/version-1.5.0/archived_design/namespace_resource_quota.md b/versioned_docs/version-1.5.0/archived_design/namespace_resource_quota.md
new file mode 100644
index 0000000..90830b6
--- /dev/null
+++ b/versioned_docs/version-1.5.0/archived_design/namespace_resource_quota.md
@@ -0,0 +1,183 @@
+---
+id: namespace_resource_quota
+title: Namespace Resource Quota
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+In K8s, user can setup namespace with [resource quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/) to limit aggregated resource consumption in this namespace. The validation of namespace resource quotas is handled in api-server directly, therefore YuniKorn simply honors the quotas like the default scheduler.
+
+## Best practice
+
+It is not mandatory to setup YuniKorn queues with respect of namespaces.
+However, in practice, it makes more sense to do so.
+Namespace is often used to set a cap for resource consumptions per user-group/team,
+YuniKorn queue is also meant to divide cluster resource into multiple groups.
+Let's go through an example.
+
+### 1. Setup namespace
+
+Namespace: `advertisement`:
+```
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+ name: advertisement
+spec:
+ hard:
+ requests.cpu: "200m"
+ requests.memory: 2000Mi
+ limits.cpu: "200m"
+ limits.memory: 4000Mi
+```
+Create the namespace
+```
+kubectl create namespace advertisement
+kubectl create -f ./advertisement.yaml --namespace=advertisement
+kubectl get quota --namespace=advertisement
+kubectl describe quota advertisement --namespace=advertisement
+
+// output
+Name: advertisement
+Namespace: advertisement
+Resource Used Hard
+-------- ---- ----
+limits.cpu 0 200m
+limits.memory 0 4000Mi
+requests.cpu 0 200m
+requests.memory 0 2000Mi
+```
+
+### 2. Setup YuniKorn queues
+
+Queue: `advertisement`:
+```
+name: advertisement
+resources:
+ guaranteed:
+ vcore: 100
+ memory: 1000
+ max:
+ vcore: 200
+ memory: 2000
+```
+
+ensure `QueueMaxResource <= NamespaceResourceQuotaRequests`
+
+### 3. Mapping applications to queues & namespace
+
+In a pod spec
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: advertisement
+ labels:
+ app: sleep
+ applicationId: "application_2019_01_22_00001"
+ queue: "root.advertisement"
+ name: task0
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-5s
+ image: "alpine:latest"
+ command: ["/bin/ash", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+ resources:
+ requests:
+ cpu: "50m"
+ memory: "800M"
+ limits:
+ cpu: "100m"
+ memory: "1000M"
+```
+
+Check Quota
+
+```
+kubectl describe quota advertisement --namespace=advertisement
+
+Name: advertisement
+Namespace: advertisement
+Resource Used Hard
+-------- ---- ----
+limits.cpu 100m 200m
+limits.memory 1G 4000Mi
+requests.cpu 50m 200m
+requests.memory 800M 2000Mi
+```
+
+Now submit another application,
+
+```
+apiVersion: v1
+kind: Pod
+metadata:
+ namespace: advertisement
+ labels:
+ app: sleep
+ applicationId: "application_2019_01_22_00002"
+ queue: "root.advertisement"
+ name: task1
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-5s
+ image: "alpine:latest"
+ command: ["/bin/ash", "-ec", "while :; do echo '.'; sleep 5 ; done"]
+ resources:
+ requests:
+ cpu: "200m"
+ memory: "800M"
+ limits:
+ cpu: "200m"
+ memory: "1000M"
+```
+
+pod will not be able to submitted to api-server, because the requested cpu `200m` + used cpu `100m` = `300m` which exceeds the resource quota.
+
+```
+kubectl create -f pod_ns_adv_task1.yaml
+Error from server (Forbidden): error when creating "pod_ns_adv_task1.yaml": pods "task1" is forbidden: exceeded quota: advertisement, requested: limits.cpu=200m,requests.cpu=200m, used: limits.cpu=100m,requests.cpu=50m, limited: limits.cpu=200m,requests.cpu=200m
+```
+
+## Future Work
+
+For compatibility, we should respect namespaces and resource quotas.
+Resource quota is overlapped with queue configuration in many ways,
+for example the `requests` quota is just like queue's max resource. However,
+there are still a few features resource quota can do but queue cannot, such as
+
+1. Resource `limits`. The aggregated resource from all pods in a namespace cannot exceed this limit.
+2. Storage Resource Quota, e.g storage size, PVC number, etc.
+3. Object Count Quotas, e.g count of PVCs, services, configmaps, etc.
+4. Resource Quota can map to priority class.
+
+Probably we can build something similar to cover (3) in this list.
+But it would be hard to completely support all these cases.
+
+But currently, setting applications mapping to a queue as well as a corresponding namespace is over complex.
+Some future improvements might be:
+
+1. Automatically detects namespaces in k8s-shim and map them to queues. Behind the scenes, we automatically generates queue configuration based on namespace definition. Generated queues are attached under root queue.
+2. When new namespace added/updated/removed, similarly to (1), we automatically update queues.
+3. User can add more configuration to queues, e.g add queue ACL, add child queues on the generated queues.
+4. Applications submitted to namespaces are transparently submitted to corresponding queues.
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/archived_design/pluggable_app_management.md b/versioned_docs/version-1.5.0/archived_design/pluggable_app_management.md
new file mode 100644
index 0000000..d297ada
--- /dev/null
+++ b/versioned_docs/version-1.5.0/archived_design/pluggable_app_management.md
@@ -0,0 +1,75 @@
+---
+id: pluggable_app_management
+title: Pluggable App Management
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## The Problem
+
+Currently, we schedule and group an application is based on a label on the pod.
+This generic way works for any type of workload. It does however give us a limited information on the lifecycle
+and application. On the K8s side, operators have been introduced to provide more detail on the application
+and help scheduling. We cannot use them currently and want to add that functionality.
+
+## K8s Operator Pattern
+
+[K8s operator](https://kubernetes.io/docs/concepts/extend-kubernetes/operator/)
+is a pattern in K8s to manage applications, it's a handy way to manage application's lifecycle out-of-box on K8s.
+You define several CRDs and some controllers to monitor and mutate the state of the application based on the CRD definition.
+
+For example in [spark-k8s-operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator),
+it defines a CRD called `SparkApplication`, the controller watches the events of add/update/delete of this CRD
+and trigger corresponding actions on event notifications. The `SparkApplication` looks like
+[this example](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/examples/spark-pi.yaml). There
+are a lot more popular operators, such as [flink-k8s-operator](https://github.com/GoogleCloudPlatform/flink-on-k8s-operator),
+ [tf-operator](https://github.com/kubeflow/tf-operator), [pytorch-operator](https://github.com/kubeflow/pytorch-operator), etc.
+
+Use Spark as an example. YuniKorn is able to schedule resources for all pods in K8s, that seamlessly supports Spark. It
+works with [native Spark on K8s](https://spark.apache.org/docs/latest/running-on-kubernetes.html), or
+[spark on K8s with operator](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/design.md#architecture),
+you'll find the difference from the design architecture chart from the given link. To support native Spark on K8s,
+YuniKorn reads pods' spec and group Spark pods by a label-selector, based on `spark-app-selector`.
+The operator approach gives us more context about the Spark job, such as a better understanding about job state.
+But all these info requires us to look at `SparkApplication` CRD, currently, there is no neat way to
+add such functionality. That's why we need to design a flexible approach to support 3rd party operators
+(retrieving info from their CRDs), so we can easily integrate with other operators with small effort.
+
+## Design
+
+The key issue here is we need a app-management interface, that can be easily extended.
+It needs to be decoupled with existing scheduling logic. For each operator, we create a service to manage this type app's lifecycle,
+and communicate with the scheduling cache independently. The high-level design looks like below:
+
+
+
+Where
+- `AppManagementService` is a composite set of services that can be managed together.
+- `AppManager` is a specific app management service for a particular type of application. In each service, it has
+ access to K8s clients, such as informers, listers, in order to monitor CRD events. And it collects necessary info
+ and talk with scheduler cache through `AMProtocol`.
+- `APIProvider` encapsulate a set of useful APIs that can be shared, such as kube-client, pod/node/storage informers, etc.
+ Each of such informers, it can be shared with multiple app managers, to avoid the overhead.
+- `AMProtocol` defines the basic interaction contract between app manager and the scheduler cache, that helps the cache
+ to performs app lifecycle management without understanding what type of the application it is.
+
+In the upon chart, the AppManagementService has 2 services, the _general_ one is managing normal applications, that
+recognizes applications by pod labels; the _spark-k8s-operator_ one watches `SparkApplication` CRD and manage jobs'
+lifecycle defined by this CRD.
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/archived_design/predicates.md b/versioned_docs/version-1.5.0/archived_design/predicates.md
new file mode 100644
index 0000000..9233a25
--- /dev/null
+++ b/versioned_docs/version-1.5.0/archived_design/predicates.md
@@ -0,0 +1,80 @@
+---
+id: predicates
+title: Support K8s Predicates
+---
+
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+## Design
+
+Predicates are a set of pre-registered functions in K8s, the scheduler invokes these functions to check if a pod
+is eligible to be allocated onto a node. Common predicates are: node-selector, pod affinity/anti-affinity etc. To support
+these predicates in YuniKorn, we don't intend to re-implement everything on our own, but to re-use the core predicates
+code as much as possible.
+
+YuniKorn-core is agnostic about underneath RMs, so the predicates functions are implemented in K8s-shim as a `SchedulerPlugin`.
+SchedulerPlugin is a way to plug/extend scheduler capabilities. Shim can implement such plugin and register itself to
+yunikorn-core, so plugged function can be invoked in the scheduler core. Find all supported plugins in
+[types](https://github.com/apache/yunikorn-core/blob/master/pkg/plugins/types.go).
+
+## Workflow
+
+First, RM needs to register itself to yunikorn-core, it advertises what scheduler plugin interfaces are supported.
+E.g a RM could implement `PredicatePlugin` interface and register itself to yunikorn-core. Then yunikorn-core will
+call PredicatePlugin API to run predicates before making allocation decisions.
+
+
+Following workflow demonstrates how allocation looks like when predicates are involved.
+
+```
+pending pods: A, B
+shim sends requests to core, including A, B
+core starts to schedule A, B
+ partition -> queue -> app -> request
+ schedule A (1)
+ run predicates (3)
+ generate predicates metadata (4)
+ run predicate functions one by one with the metadata
+ success
+ proposal: A->N
+ schedule B (2)
+ run predicates (calling shim API)
+ generate predicates metadata
+ run predicate functions one by one with the metadata
+ success
+ proposal: B->N
+commit the allocation proposal for A and notify k8s-shim
+commit the allocation proposal for B and notify k8s-shim
+shim binds pod A to N
+shim binds pod B to N
+```
+
+(1) and (2) are running in parallel.
+
+(3) yunikorn-core calls a `schedulerPlugin` API to run predicates, this API is implemented on k8s-shim side.
+
+(4) K8s-shim generates metadata based on current scheduler cache, the metadata includes some intermittent states about nodes and pods.
+
+## Predicates White-list
+
+Intentionally, we only support a white-list of predicates. Majorly due to 2 reasons,
+* Predicate functions are time-consuming, it has negative impact on scheduler performance. To support predicates that are only necessary can minimize the impact. This will be configurable via CLI options;
+* The implementation depends heavily on K8s default scheduler code, though we reused some unit tests, the coverage is still a problem. We'll continue to improve the coverage when adding new predicates.
+
+The white-list currently is defined in [PredicateManager](https://github.com/apache/yunikorn-k8shim/blob/master/pkg/plugin/predicates/predicate_manager.go).
diff --git a/versioned_docs/version-1.5.0/archived_design/scheduler_core_design.md b/versioned_docs/version-1.5.0/archived_design/scheduler_core_design.md
new file mode 100644
index 0000000..c3da548
--- /dev/null
+++ b/versioned_docs/version-1.5.0/archived_design/scheduler_core_design.md
@@ -0,0 +1,401 @@
+---
+id: scheduler_core_design
+title: Scheduler Core Design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+:::caution
+The scheduler core design has changed. [YUNIKORN-317](https://issues.apache.org/jira/browse/YUNIKORN-317) was committed and has removed the scheduler cache.
+This document will not be maintained and is just for historical reference.
+See [scheduler cache removal design](../design/cache_removal.md)
+:::
+
+Github repo: https://github.com/apache/yunikorn-core
+
+Scheduler core encapsulates all scheduling algorithms, it collects resources from underneath resource management
+platforms (like YARN/K8s), and is responsible for container allocation requests. It makes the decision where is the
+best spot for each request and then sends response allocations to the resource management platform.
+Scheduler core is agnostic about underneath platforms, all the communications are through the [scheduler interface](https://github.com/apache/yunikorn-scheduler-interface).
+
+## Components:
+
+```
+
+ +---------------+ +--------------+
+ |K8s Shim | |YARN Shim |
+ +---------------+ +--------------+
+
+ +--------------+ +------------+
+ Scheduler- | GRPC Protocol| |Go API |
+ Interface: +--------------+ +------------+
+
++---------------------------------------------------------------------------+
+ +--------------------+
+ |Scheduler API Server|
+ +-------------+ +---------+----------+
+ |AdminService | |
+ +-------------+ |Write Ops +----------------+
+ +-------------+ V ++Scheduler |
+ |Configurator | +-------------------+ Allocate || And |
+ +-------------+ |Cache Event Handler+<-----------------| |
+ +----------> +-------------------+ Preempt ++Preemptor |
+ Update Cfg Handled by policies +----------------+
+ + (Stateless)
+ +------v--------+
+ |Scheduler Cache|
+ +---------------+
+ +---------------------------------------------+
+ |--------+ +------+ +----------+ +----------+ |
+ ||Node | |Queue | |Allocation| |Requests | |
+ |--------+ +------+ +----------+ +----------+ |
+ +---------------------------------------------+
+```
+
+### Scheduler API Server (RMProxy)
+
+Responsible for communication between RM and Scheduler, which implements scheduler-interface GRPC protocol,
+or just APIs. (For intra-process communication w/o Serde).
+
+### Scheduler Cache
+
+Caches all data related to scheduler state, such as used resources of each queues, nodes, allocations.
+Relationship between allocations and nodes, etc. Should not include in-flight data for resource allocation.
+For example to-be-preempted allocation candidates. Fair share resource of queues, etc.
+
+### Scheduler Cache Event Handler
+
+Handles all events which needs to update scheduler internal state. So all the write operations will be carefully handled.
+
+### Admin Service
+
+Handles request from Admin, which can also load configurations from storage and update scheduler policies.
+
+### Scheduler and Preemptor
+
+Handles Scheduler's internal state. (Which is not belong to scheduelr cache), such as internal reservations, etc.
+Scheduler and preemptor will work together, make scheduling or preemption decisions. All allocate/preempt request
+will be handled by event handler.
+
+## Scheduler's responsibility
+
+- According to resource usages between queues, sort queues, applications, and figure out order of application allocation. (This will be used by preemption as well).
+- It is possible that we cannot satisfy some of the allocation request, we need to skip them and find next request.
+- It is possible that some allocation request cannot be satisfied because of resource fragmentation. We need to reserve room for such requests.
+- Different nodes may belong to different disjoint partitions, we can make independent scheduler runs
+- Be able to config and change ordering policies for apps, queues.
+- Application can choose their own way to manage sort of nodes.
+
+## Preemption
+
+- It is important to know "who wants the resource", so we can do preemption based on allocation orders.
+- When do preemption, it is also efficient to trigger allocation op. Think about how to do it.
+- Preemption needs to take care about queue resource balancing.
+
+## Communication between Shim and Core
+
+YuniKorn-Shim (like https://github.com/apache/yunikorn-k8shim) communicates with core by
+using scheduler-interface (https://github.com/apache/yunikorn-scheduler-interface).
+Scheduler interface has Go API or GRPC. Currently, yunikorn-k8shim is using Go API to communicate with yunikorn-core
+to avoid extra overhead introduced by GRPC.
+
+**Shim (like K8shim) first need to register with core:**
+
+```go
+func (m *RMProxy) RegisterResourceManager(request *si.RegisterResourceManagerRequest, callback api.ResourceManagerCallback) (*si.RegisterResourceManagerResponse, error)
+```
+
+Which indicate ResourceManager's name, a callback function for updateResponse. The design of core is be able to do scheduling for multiple clusters (such as multiple K8s cluster) just with one core instance.
+
+**Shim interacts with core by invoking RMProxy's Update API frequently, which updates new allocation request, allocation to kill, node updates, etc.**
+
+```go
+func (m *RMProxy) Update(request *si.UpdateRequest) error
+```
+
+Response of update (such as new allocated container) will be received by registered callback.
+
+## Configurations & Semantics
+
+Example of configuration:
+
+- Partition is name space.
+- Same queues can under different partitions, but enforced to have same hierarchy.
+
+ Good:
+
+ ```
+ partition=x partition=y
+ a a
+ / \ / \
+ b c b c
+ ```
+
+ Good (c in partition y acl=""):
+
+ ```
+ partition=x partition=y
+ a a
+ / \ /
+ b c b
+ ```
+
+ Bad (c in different hierarchy)
+
+ ```
+ partition=x partition=y
+ a a
+ / \ / \
+ b c b d
+ /
+ c
+ ```
+
+ Bad (Duplicated c)
+
+ ```
+ partition=x
+ a
+ / \
+ b c
+ /
+ c
+
+ ```
+
+- Different hierarchies can be added
+
+ ```scheduler-conf.yaml
+ partitions:
+ - name: default
+ queues:
+ root:
+ configs:
+ acls:
+ childrens:
+ - a
+ - b
+ - c
+ - ...
+ a:
+ configs:
+ acls:
+ capacity: (capacity is not allowed to set for root)
+ max-capacity: ...
+ mapping-policies:
+ ...
+ - name: partition_a:
+ queues:
+ root:...
+ ```
+
+## How scheduler do allocation
+
+Scheduler runs a separate goroutine to look at asks and available resources, and do resource allocation. Here's allocation logic in pseudo code:
+
+Entry point of scheduler allocation is `scheduler.go: func (s *Scheduler) schedule()`
+
+```
+# First of all, YuniKorn has partition concept, a logical resource pool can consists
+# of one of multiple physical dis-joint partitions. It is similar to YARN's node
+# partition concept.
+
+for partition : partitions:
+ # YuniKorn can reserve allocations for picky asks (such as large request, etc.)
+ # Before doing regular allocation, YuniKorn look at reservedAllocations first.
+ for reservedAllocation : partition.reservedAllocations:
+ reservedAllocation.tryAllocate(..)
+
+ # After tried all reserved allocation, YuniKorn will go to regular allocation
+ partition.tryAllocate(..)
+
+ # If there's any allocation created, scheduler will create an AllocationProposal
+ # and send to Cache to "commit" the AllocationProposal
+```
+
+**Allocation by hierchical of queues**
+
+Inside `partition.tryAllocate`
+
+It recursively traverse from root queue and down to lower level, for each level, logic is inside `pkg/scheduler/scheduling_queue.go func (sq *SchedulingQueue) tryAllocate`
+
+Remember YuniKorn natively supports hierarchical of queues. For ParentQueue (which has sub queues under the parent queue), it uses queue's own sorting policy to sort subqueues and try to allocate from most preferred queue to least-preferred queue.
+
+For LeafQueue (which has applications inside the queue), it uses queue's own sorting policy to sort applications belong to the queue and allocate based on the sorted order.
+
+(All sorting policies can be configured differently at each level.)
+
+**Allocation by application**
+
+When it goes to Application, see (`scheduler_application.go: func (sa *SchedulingApplication) tryAllocate`), It first sort the pending resource requests belong to the application (based on requests' priority). And based on the selected request, and configured node-sorting policy, it sorts nodes belong to the partition and try to allocate resources on the sorted nodes.
+
+When application trying to allocate resources on nodes, it will invokes PredicatePlugin to make sure Shim can confirm the node is good. (For example K8shim runs predicates check for allocation pre-check).
+
+**Allocation completed by scheduler**
+
+Once allocation is done, scheduler will create an AllocationProposal and send to Cache to do further check, we will cover details in the upcoming section.
+
+## Flow of events
+
+Like mentioned before, all communications between components like RMProxy/Cache/Schedulers are done by using async event handler.
+
+RMProxy/Cache/Scheduler include local event queues and event handlers. RMProxy and Scheduler have only one queue (For example: `pkg/scheduler/scheduler.go: handleSchedulerEvent`), and Cache has two queues (One for events from RMProxy, and one for events from Scheduler, which is designed for better performance).
+
+We will talk about how events flowed between components:
+
+**Events for ResourceManager registration and updates:**
+
+```
+Update from ResourceManager -> RMProxy -> RMUpdateRequestEvent Send to Cache
+New ResourceManager registration -> RMProxy -> RegisterRMEvent Send to Cache
+```
+
+**Cache Handles RM Updates**
+
+There're many fields inside RM Update event (`RMUpdateRequestEvent`), among them, we have following categories:
+
+```
+1) Update for Application-related updates
+2) Update for New allocation ask and release.
+3) Node (Such as kubelet) update (New node, remove node, node resource change, etc.)
+```
+
+More details can be found at:
+
+```
+func (m *ClusterInfo) processRMUpdateEvent(event *cacheevent.RMUpdateRequestEvent)
+
+inside cluster_info.go
+```
+
+**Cache send RM updates to Scheduler**
+
+For most cases, Cache propagate updates from RM to scheduler directly (including Application, Node, Asks, etc.). And it is possible that some updates from RM is not valid (such as adding an application to a non-existed queue), for such cases, cache can send an event back to RMProxy and notify the ResourceManager. (See `RMApplicationUpdateEvent.RejectedApplications`)
+
+**Cache handles scheduler config**
+
+Cache also handles scheduler's config changes, see
+
+```go
+func (m *ClusterInfo) processRMConfigUpdateEvent(event *commonevents.ConfigUpdateRMEvent)
+```
+
+Similar to other RM updates, it propages news to scheduelr.
+
+**Scheduler do allocation**
+
+Once an AllocationProposal created by scheduler, scheduler sends `AllocationProposalBundleEvent` to Cache to commit.
+
+Cache look at AllocationProposal under lock, and commit these proposals. The reason to do proposal/commit is Scheduler can run in multi-threads which could cause conflict for resource allocation. This approach is inspired by Borg/Omega/YARN Global Scheduling.
+
+Cache checks more states such as queue resources, node resources (we cannot allocate more resource than nodes' available), etc. Once check is done, Cache updates internal data strcture and send confirmation to Scheduler to update the same, and scheduler sends allocated Allocation to RMProxy so Shim can do further options. For example, K8shim will `bind` an allocation (POD) to kubelet.
+
+```
+Job Add:
+--------
+RM -> Cache -> Scheduler (Implemented)
+
+Job Remove:
+-----------
+RM -> Scheduler -> Cache (Implemented)
+Released allocations: (Same as normal release) (Implemented)
+Note: Make sure remove from scheduler first to avoid new allocated created.
+
+Scheduling Request Add:
+-----------------------
+RM -> Cache -> Scheduler (Implemented)
+Note: Will check if requested job exists, queue exists, etc.
+When any request invalid:
+ Cache -> RM (Implemented)
+ Scheduler -> RM (Implemented)
+
+Scheduling Request remove:
+-------------------------
+RM -> Scheduler -> Cache (Implemented)
+Note: Make sure removed from scheduler first to avoid new container allocated
+
+Allocation remove (Preemption)
+-----------------
+Scheduler -> Cache -> RM (TODO)
+ (confirmation)
+
+Allocation remove (RM voluntarilly ask)
+---------------------------------------
+RM -> Scheduler -> Cache -> RM. (Implemented)
+ (confirmation)
+
+Node Add:
+---------
+RM -> Cache -> Scheduler (Implemented)
+Note: Inside Cache, update allocated resources.
+Error handling: Reject Node to RM (Implemented)
+
+Node Remove:
+------------
+Implemented in cache side
+RM -> Scheduler -> Cache (TODO)
+
+Allocation Proposal:
+--------------------
+Scheduler -> Cache -> RM
+When rejected/accepted:
+ Cache -> Scheduler
+
+Initial: (TODO)
+--------
+1. Admin configured partitions
+2. Cache initializes
+3. Scheduler copies configurations
+
+Relations between Entities
+-------------------------
+1. RM includes one or multiple:
+ - Partitions
+ - Jobs
+ - Nodes
+ - Queues
+
+2. One queue:
+ - Under one partition
+ - Under one RM.
+
+3. One job:
+ - Under one queue (Job with same name can under different partitions)
+ - Under one partition
+
+RM registration: (TODO)
+----------------
+1. RM send registration
+2. If RM already registered, remove old one, including everything belong to RM.
+
+RM termination (TODO)
+--------------
+Just remove the old one.
+
+Update of queues (TODO)
+------------------------
+Admin Service -> Cache
+
+About partition (TODO)
+-----------------------
+Internal partition need to be normalized, for example, RM specify node with partition = xyz.
+Scheduler internally need to normalize it to <rm-id>_xyz
+This need to be done by RMProxy
+
+```
diff --git a/versioned_docs/version-1.5.0/assets/RunningSparkOnK8s.png b/versioned_docs/version-1.5.0/assets/RunningSparkOnK8s.png
new file mode 100644
index 0000000..7594a07
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/RunningSparkOnK8s.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/allocation_4k.png b/versioned_docs/version-1.5.0/assets/allocation_4k.png
new file mode 100644
index 0000000..03346f5
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/allocation_4k.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/application-state.png b/versioned_docs/version-1.5.0/assets/application-state.png
new file mode 100644
index 0000000..2af46e9
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/application-state.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/architecture.png b/versioned_docs/version-1.5.0/assets/architecture.png
new file mode 100644
index 0000000..a19dcaa
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/architecture.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/cpu_profile.jpg b/versioned_docs/version-1.5.0/assets/cpu_profile.jpg
new file mode 100644
index 0000000..7e99f62
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/cpu_profile.jpg
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/dashboard_secret.png b/versioned_docs/version-1.5.0/assets/dashboard_secret.png
new file mode 100644
index 0000000..60b4f97
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/dashboard_secret.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/dashboard_token_select.png b/versioned_docs/version-1.5.0/assets/dashboard_token_select.png
new file mode 100644
index 0000000..59173fd
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/dashboard_token_select.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/docker-dektop-minikube.png b/versioned_docs/version-1.5.0/assets/docker-dektop-minikube.png
new file mode 100644
index 0000000..48b3584
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/docker-dektop-minikube.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/docker-desktop.png b/versioned_docs/version-1.5.0/assets/docker-desktop.png
new file mode 100644
index 0000000..9224360
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/docker-desktop.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/external_userinfo.png b/versioned_docs/version-1.5.0/assets/external_userinfo.png
new file mode 100644
index 0000000..1598351
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/external_userinfo.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/fifo-state-example.png b/versioned_docs/version-1.5.0/assets/fifo-state-example.png
new file mode 100644
index 0000000..ca04c17
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/fifo-state-example.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/gang_clean_up.png b/versioned_docs/version-1.5.0/assets/gang_clean_up.png
new file mode 100644
index 0000000..baf5acc
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/gang_clean_up.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/gang_generic_flow.png b/versioned_docs/version-1.5.0/assets/gang_generic_flow.png
new file mode 100644
index 0000000..381b7d0
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/gang_generic_flow.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/gang_scheduling_intro.png b/versioned_docs/version-1.5.0/assets/gang_scheduling_intro.png
new file mode 100644
index 0000000..b3be207
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/gang_scheduling_intro.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/gang_timeout.png b/versioned_docs/version-1.5.0/assets/gang_timeout.png
new file mode 100644
index 0000000..a8ea0da
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/gang_timeout.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/gang_total_ask.png b/versioned_docs/version-1.5.0/assets/gang_total_ask.png
new file mode 100644
index 0000000..928d7fd
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/gang_total_ask.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/goland_debug.jpg b/versioned_docs/version-1.5.0/assets/goland_debug.jpg
new file mode 100644
index 0000000..c9ab94c
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/goland_debug.jpg
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/goland_debug.png b/versioned_docs/version-1.5.0/assets/goland_debug.png
new file mode 100644
index 0000000..aa8803f
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/goland_debug.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/grafana_dashboard.png b/versioned_docs/version-1.5.0/assets/grafana_dashboard.png
new file mode 100644
index 0000000..b097ee3
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/grafana_dashboard.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/grafana_login_page.png b/versioned_docs/version-1.5.0/assets/grafana_login_page.png
new file mode 100644
index 0000000..9aa818b
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/grafana_login_page.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/import_dashboard_01.png b/versioned_docs/version-1.5.0/assets/import_dashboard_01.png
new file mode 100644
index 0000000..60d51d8
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/import_dashboard_01.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/import_dashboard_02.png b/versioned_docs/version-1.5.0/assets/import_dashboard_02.png
new file mode 100644
index 0000000..bbf18ab
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/import_dashboard_02.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/k8shim-application-state.png b/versioned_docs/version-1.5.0/assets/k8shim-application-state.png
new file mode 100644
index 0000000..c286591
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/k8shim-application-state.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/k8shim-node-state.png b/versioned_docs/version-1.5.0/assets/k8shim-node-state.png
new file mode 100644
index 0000000..5d5db56
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/k8shim-node-state.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/k8shim-task-state.png b/versioned_docs/version-1.5.0/assets/k8shim-task-state.png
new file mode 100644
index 0000000..11c95d1
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/k8shim-task-state.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/kwok-affinity.png b/versioned_docs/version-1.5.0/assets/kwok-affinity.png
new file mode 100644
index 0000000..a79d935
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/kwok-affinity.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/kwok-priorityclass.png b/versioned_docs/version-1.5.0/assets/kwok-priorityclass.png
new file mode 100644
index 0000000..e704409
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/kwok-priorityclass.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/kwok-throughput-01.png b/versioned_docs/version-1.5.0/assets/kwok-throughput-01.png
new file mode 100644
index 0000000..59e34a5
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/kwok-throughput-01.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/kwok-throughput-02.png b/versioned_docs/version-1.5.0/assets/kwok-throughput-02.png
new file mode 100644
index 0000000..3e186a9
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/kwok-throughput-02.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/kwok-throughput-03.png b/versioned_docs/version-1.5.0/assets/kwok-throughput-03.png
new file mode 100644
index 0000000..6b87e32
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/kwok-throughput-03.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/kwok-toleration-taint.png b/versioned_docs/version-1.5.0/assets/kwok-toleration-taint.png
new file mode 100644
index 0000000..183c721
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/kwok-toleration-taint.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/namespace-mapping.png b/versioned_docs/version-1.5.0/assets/namespace-mapping.png
new file mode 100644
index 0000000..9ad07da
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/namespace-mapping.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/node-bin-packing.png b/versioned_docs/version-1.5.0/assets/node-bin-packing.png
new file mode 100644
index 0000000..9267a03
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/node-bin-packing.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/node-fair.png b/versioned_docs/version-1.5.0/assets/node-fair.png
new file mode 100644
index 0000000..2e404ab
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/node-fair.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/node_fairness_conf.png b/versioned_docs/version-1.5.0/assets/node_fairness_conf.png
new file mode 100644
index 0000000..80e1dd5
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/node_fairness_conf.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/object-state.png b/versioned_docs/version-1.5.0/assets/object-state.png
new file mode 100644
index 0000000..9baca07
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/object-state.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/perf-tutorial-build.png b/versioned_docs/version-1.5.0/assets/perf-tutorial-build.png
new file mode 100644
index 0000000..5c2f28b
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/perf-tutorial-build.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/perf-tutorial-resultDiagrams.png b/versioned_docs/version-1.5.0/assets/perf-tutorial-resultDiagrams.png
new file mode 100644
index 0000000..6d9686d
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/perf-tutorial-resultDiagrams.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/perf-tutorial-resultLog.png b/versioned_docs/version-1.5.0/assets/perf-tutorial-resultLog.png
new file mode 100755
index 0000000..dad58bf
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/perf-tutorial-resultLog.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/perf_e2e_test.png b/versioned_docs/version-1.5.0/assets/perf_e2e_test.png
new file mode 100644
index 0000000..79763d0
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/perf_e2e_test.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/perf_e2e_test_conf.png b/versioned_docs/version-1.5.0/assets/perf_e2e_test_conf.png
new file mode 100644
index 0000000..5aeef94
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/perf_e2e_test_conf.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/perf_node_fairness.png b/versioned_docs/version-1.5.0/assets/perf_node_fairness.png
new file mode 100644
index 0000000..fbacc13
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/perf_node_fairness.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/perf_throughput.png b/versioned_docs/version-1.5.0/assets/perf_throughput.png
new file mode 100644
index 0000000..e89a7f3
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/perf_throughput.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/pluggable-app-mgmt.jpg b/versioned_docs/version-1.5.0/assets/pluggable-app-mgmt.jpg
new file mode 100644
index 0000000..443b8ad
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/pluggable-app-mgmt.jpg
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/predicateComaparation.png b/versioned_docs/version-1.5.0/assets/predicateComaparation.png
new file mode 100755
index 0000000..d3498c8
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/predicateComaparation.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/predicate_4k.png b/versioned_docs/version-1.5.0/assets/predicate_4k.png
new file mode 100644
index 0000000..850036c
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/predicate_4k.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/preemption_fence.png b/versioned_docs/version-1.5.0/assets/preemption_fence.png
new file mode 100644
index 0000000..61222bc
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/preemption_fence.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/preemption_fence_case1.png b/versioned_docs/version-1.5.0/assets/preemption_fence_case1.png
new file mode 100644
index 0000000..cb8f7b4
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/preemption_fence_case1.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/preemption_fence_case2.png b/versioned_docs/version-1.5.0/assets/preemption_fence_case2.png
new file mode 100644
index 0000000..6156ebf
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/preemption_fence_case2.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/preemption_fence_hierarchy_case.png b/versioned_docs/version-1.5.0/assets/preemption_fence_hierarchy_case.png
new file mode 100644
index 0000000..26db9c7
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/preemption_fence_hierarchy_case.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/preemption_normal_case.png b/versioned_docs/version-1.5.0/assets/preemption_normal_case.png
new file mode 100644
index 0000000..1864422
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/preemption_normal_case.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/preemption_priority_queue_case.png b/versioned_docs/version-1.5.0/assets/preemption_priority_queue_case.png
new file mode 100644
index 0000000..6d6fb13
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/preemption_priority_queue_case.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/preemption_priorityclass_case.png b/versioned_docs/version-1.5.0/assets/preemption_priorityclass_case.png
new file mode 100644
index 0000000..91676b0
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/preemption_priorityclass_case.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/priority-flows.png b/versioned_docs/version-1.5.0/assets/priority-flows.png
new file mode 100644
index 0000000..19b82ec
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/priority-flows.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/priority-tree.png b/versioned_docs/version-1.5.0/assets/priority-tree.png
new file mode 100644
index 0000000..7288db5
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/priority-tree.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/prometheus-cmd.png b/versioned_docs/version-1.5.0/assets/prometheus-cmd.png
new file mode 100644
index 0000000..86bfc6a
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/prometheus-cmd.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/prometheus-web-ui.png b/versioned_docs/version-1.5.0/assets/prometheus-web-ui.png
new file mode 100644
index 0000000..b451976
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/prometheus-web-ui.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/prometheus.png b/versioned_docs/version-1.5.0/assets/prometheus.png
new file mode 100644
index 0000000..964c79c
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/prometheus.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/queue-fairness.png b/versioned_docs/version-1.5.0/assets/queue-fairness.png
new file mode 100644
index 0000000..7c78ed7
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/queue-fairness.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/queue-resource-quotas.png b/versioned_docs/version-1.5.0/assets/queue-resource-quotas.png
new file mode 100644
index 0000000..fb72138
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/queue-resource-quotas.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/quota_check.png b/versioned_docs/version-1.5.0/assets/quota_check.png
new file mode 100644
index 0000000..b8d29e6
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/quota_check.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/resilience-node-recovery.jpg b/versioned_docs/version-1.5.0/assets/resilience-node-recovery.jpg
new file mode 100644
index 0000000..3847451
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/resilience-node-recovery.jpg
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/resilience-workflow.jpg b/versioned_docs/version-1.5.0/assets/resilience-workflow.jpg
new file mode 100644
index 0000000..40ab6ba
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/resilience-workflow.jpg
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/scheduling_no_predicate_4k.png b/versioned_docs/version-1.5.0/assets/scheduling_no_predicate_4k.png
new file mode 100644
index 0000000..0ebe41c
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/scheduling_no_predicate_4k.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/scheduling_with_predicate_4k_.png b/versioned_docs/version-1.5.0/assets/scheduling_with_predicate_4k_.png
new file mode 100644
index 0000000..2cee7c0
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/scheduling_with_predicate_4k_.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/simple_preemptor.png b/versioned_docs/version-1.5.0/assets/simple_preemptor.png
new file mode 100644
index 0000000..c5165c3
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/simple_preemptor.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/sparkResult.png b/versioned_docs/version-1.5.0/assets/sparkResult.png
new file mode 100644
index 0000000..8a11e9b
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/sparkResult.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/tf-job-gpu-on-logs.png b/versioned_docs/version-1.5.0/assets/tf-job-gpu-on-logs.png
new file mode 100644
index 0000000..db2a6b6
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/tf-job-gpu-on-logs.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/tf-job-gpu-on-ui.png b/versioned_docs/version-1.5.0/assets/tf-job-gpu-on-ui.png
new file mode 100644
index 0000000..b599dca
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/tf-job-gpu-on-ui.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/tf-job-on-ui.png b/versioned_docs/version-1.5.0/assets/tf-job-on-ui.png
new file mode 100644
index 0000000..06acabe
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/tf-job-on-ui.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/throughput.png b/versioned_docs/version-1.5.0/assets/throughput.png
new file mode 100644
index 0000000..8ced22c
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/throughput.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/throughput_3types.png b/versioned_docs/version-1.5.0/assets/throughput_3types.png
new file mode 100644
index 0000000..a4a583b
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/throughput_3types.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/throughput_conf.png b/versioned_docs/version-1.5.0/assets/throughput_conf.png
new file mode 100644
index 0000000..dd5d720
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/throughput_conf.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/translationBuilding.png b/versioned_docs/version-1.5.0/assets/translationBuilding.png
new file mode 100644
index 0000000..0ba7250
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/translationBuilding.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/translationDemo.png b/versioned_docs/version-1.5.0/assets/translationDemo.png
new file mode 100644
index 0000000..4125dc6
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/translationDemo.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/ug_lookup.png b/versioned_docs/version-1.5.0/assets/ug_lookup.png
new file mode 100644
index 0000000..d86ae6a
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/ug_lookup.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/yk-ui-screenshots.gif b/versioned_docs/version-1.5.0/assets/yk-ui-screenshots.gif
new file mode 100644
index 0000000..5ce27e1
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/yk-ui-screenshots.gif
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/yunikorn-gpu-time-slicing.png b/versioned_docs/version-1.5.0/assets/yunikorn-gpu-time-slicing.png
new file mode 100644
index 0000000..8b3d734
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/yunikorn-gpu-time-slicing.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/assets/yunirkonVSdefault.png b/versioned_docs/version-1.5.0/assets/yunirkonVSdefault.png
new file mode 100755
index 0000000..a123ff8
--- /dev/null
+++ b/versioned_docs/version-1.5.0/assets/yunirkonVSdefault.png
Binary files differ
diff --git a/versioned_docs/version-1.5.0/design/architecture.md b/versioned_docs/version-1.5.0/design/architecture.md
new file mode 100644
index 0000000..5163d4c
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/architecture.md
@@ -0,0 +1,62 @@
+---
+id: architecture
+title: Architecture
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Apache YuniKorn is a light-weight, universal resource scheduler for container orchestrator systems.
+It is created to achieve fine-grained resource sharing for various workloads efficiently on a large scale, multi-tenant,
+and cloud-native environment. YuniKorn brings a unified, cross-platform, scheduling experience for mixed workloads that
+consist of stateless batch workloads and stateful services.
+
+YuniKorn now supports K8s and can be deployed as a custom K8s scheduler. YuniKorn's architecture design also allows
+adding different shim layer and adopt to different ResourceManager implementation including Apache Hadoop YARN,
+or any other systems.
+
+## Architecture
+
+Following chart illustrates the high-level architecture of YuniKorn.
+
+<img src={require('./../assets/architecture.png').default} />
+
+## Components
+
+### Scheduler interface
+
+[Scheduler interface](https://github.com/apache/yunikorn-scheduler-interface) is an abstract layer
+which resource management platform (like YARN/K8s) will speak with, via API like GRPC/programing language bindings.
+
+### Scheduler core
+
+Scheduler core encapsulates all scheduling algorithms, it collects resources from underneath resource management
+platforms (like YARN/K8s), and is responsible for container allocation requests. It makes the decision where is the
+best spot for each request and then sends response allocations to the resource management platform.
+Scheduler core is agnostic about underneath platforms, all the communications are through the [scheduler interface](https://github.com/apache/yunikorn-scheduler-interface).
+Please read more about the design of schedule core [here](../archived_design/scheduler_core_design.md).
+
+### Kubernetes shim
+
+The YuniKorn Kubernetes shim is responsible for talking to Kubernetes, it is responsible for translating the Kubernetes
+cluster resources, and resource requests via scheduler interface and send them to the scheduler core.
+And when a scheduler decision is made, it is responsible for binding the pod to the specific node. All the communication
+between the shim and the scheduler core is through the [scheduler interface](https://github.com/apache/yunikorn-scheduler-interface).
+Please read more about the design of the Kubernetes shim [here](../archived_design/k8shim.md).
+
diff --git a/versioned_docs/version-1.5.0/design/cache_removal.md b/versioned_docs/version-1.5.0/design/cache_removal.md
new file mode 100644
index 0000000..2312643
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/cache_removal.md
@@ -0,0 +1,456 @@
+---
+id: cache_removal
+title: Scheduler cache removal design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+:::caution
+The Interface Message definitions described in this design doc has undergone major refactoring to reduce the complexity. [YUNIKORN-337](https://issues.apache.org/jira/browse/YUNIKORN-337) was committed and simplified the message communication between Core and Shim to greater extent.
+See [Simplifying Interface Messages and Breaking Shim build dependency on Core](interface_message_simplification.md) to know the updated message definitions.
+:::
+
+# Proposal to combine Cache and Scheduler's implementation in the core
+This document describes the current state of the scheduler and cache implementation.
+It describes the changes planned based on the analysis that was done of the current behaviour.
+
+## Goals
+The goal is to provide the same functionality before and after the change.
+- Unit tests before and after the merge must all pass.
+- Smoke tests defined in the core should all pass without major changes.
+- End-to-end tests that are part of the shim code must all pass without changes.
+
+:::info
+ Major changes for smoke tests are defined as changes to the tests that alter use case and thus test flows. Some changes will be needed as checks made could rely on cache objects which have been removed.
+:::
+
+## Background
+The current Scheduler Core is build up around two major components to store the data: the cache and scheduler objects.
+The cache objects form the base for most data to be tracked.
+The Scheduler objects track specific in flight details and are build on top of a cache object.
+
+The communication between the two layers uses a-synchronous events and in some cases direct updates.
+A synchronous update between the scheduler and the cache does mean that there is a short period the scheduler is "out of sync" with the cache.
+This short period can have an impact on the scheduling decisions.
+One of which is logged as [YUNIKORN-169](https://issues.apache.org/jira/browse/YUNIKORN-169).
+
+A further point is the complexity that the two structure brings to the code.
+A distinct set of messages to communicate between the scheduler and the cache. A one-on-one mapping between the scheduler and cache objects shows that the distinction is probably more artificial than required.
+
+
+---
+
+## Structure analysis
+### Objects
+The existing objects as per the code analysis.
+The overlap between the scheduler and the cache objects is shown by showing them at the same line.
+N/A means that there is no equivalent object in either the scheduler or cache.
+
+| Cache Object | Scheduler Object |
+| ------------------------------ | ------------------------------ |
+| ClusterInfo | ClusterSchedulingContext |
+| PartitionInfo | partitionSchedulingContext |
+| AllocationInfo | schedulingAllocation |
+| N/A | schedulingAllocationAsk |
+| N/A | reservation |
+| ApplicationInfo | SchedulingApplication |
+| applicationState | N/A |
+| NodeInfo | SchedulingNode |
+| QueueInfo | SchedulingQueue |
+| SchedulingObjectState | N/A |
+
+The `initializer` code that is part of the cache does not define a specific object.
+It contains a mixture of code defined at the package level and code that is part of the `ClusterInfo` object.
+
+### Events
+Events defined in the core have multiple origins and destinations.
+Some events are only internal for the core between the cache and scheduler.
+These events will be removed.
+
+| Event | Flow | Proposal |
+| ----------------------------------------- | --------------------- | -------- |
+| AllocationProposalBundleEvent | Scheduler -> Cache | Remove |
+| RejectedNewApplicationEvent | Scheduler -> Cache | Remove |
+| ReleaseAllocationsEvent | Scheduler -> Cache | Remove |
+| RemoveRMPartitionsEvent | Scheduler -> Cache | Remove |
+| RemovedApplicationEvent | Scheduler -> Cache | Remove |
+| SchedulerNodeEvent | Cache -> Scheduler | Remove |
+| SchedulerAllocationUpdatesEvent | Cache -> Scheduler | Remove |
+| SchedulerApplicationsUpdateEvent | Cache -> Scheduler | Remove |
+| SchedulerUpdatePartitionsConfigEvent | Cache -> Scheduler | Remove |
+| SchedulerDeletePartitionsConfigEvent | Cache -> Scheduler | Remove |
+| RMApplicationUpdateEvent (add/remove app) | Cache/Scheduler -> RM | Modify |
+| RMRejectedAllocationAskEvent | Cache/Scheduler -> RM | Modify |
+| RemoveRMPartitionsEvent | RM -> Scheduler | |
+| RMUpdateRequestEvent | RM -> Cache | Modify |
+| RegisterRMEvent | RM -> Cache | Modify |
+| ConfigUpdateRMEvent | RM -> Cache | Modify |
+| RMNewAllocationsEvent | Cache -> RM | Modify |
+| RMReleaseAllocationEvent | Cache -> RM | Modify |
+| RMNodeUpdateEvent | Cache -> RM | Modify |
+| | | |
+
+Events that are handled by the cache will need to be handled by the core code after the removal of the cache.
+Two events are handled by the cache and the scheduler.
+
+## Detailed flow analysis
+### Object existing in both cache and scheduler
+The current design is based on the fact that the cache object is the basis for all data storage.
+Each cache object must have a corresponding scheduler object.
+The contract in the core around the cache and scheduler objects was simple.
+If the object exists in both scheduler and cache the object will be added to cache triggering the creation of the corresponding scheduler object.
+Removing the object is always handled in reverse: first from the scheduler which will trigger the removal from the cache.
+An example would be the creation of an application triggered by the `RMUpdateRequestEvent` would be processed by the cache.
+Creating a `SchedulerApplicationsUpdateEvent` to create the corresponding application in the scheduler.
+
+When the application and object state were added they were added into the cache objects.
+The cache objects were considered the data store and thus also contain the state.
+There were no corresponding state objects in the scheduler.
+Maintaining two states for the same object is not possible.
+
+The other exceptions to that rule are two objects that were considered volatile and scheduler only.
+The `schedulingAllocationAsk` tracks outstanding requests for an application in the scheduler.
+The `reservation` tracks a temporary reservation of a node for an application and ask combination.
+
+### Operations to add/remove app
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request an application to be added or removed.
+
+**application add or delete**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processApplicationUpdateFromRMUpdate
+ 2.1: Add new apps to the partition.
+ 2.2: Send removed apps to scheduler (but not remove anything from cache)
+3. scheduler.processApplicationUpdateEvent
+ 3.1: Add new apps to scheduler
+ (when fails, send RejectedNewApplicationEvent to cache)
+ No matter if failed or not, send RMApplicationUpdateEvent to RM.
+ 3.2: Remove app from scheduler
+ Send RemovedApplicationEvent to cache
+```
+
+### Operations to remove allocations and add or remove asks
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request an allocation to be removed.
+The RM can request an ask to be added or removed
+
+**allocation delete**
+This describes the allocation delete initiated by the RM only
+````
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: (by-pass): Send to scheduler via event SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: Update ReconcilePlugin
+ 3.2: Send confirmation of the releases back to Cache via event ReleaseAllocationsEvent
+4. cluster_info.processAllocationReleases to process the confirmed release
+````
+
+**ask add**
+If the ask already exists this add is automatically converted into an update.
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: Ask sanity check (such as existence of partition/app), rejections are send back to the RM via RMRejectedAllocationAskEvent
+ 2.2: pass checked asks to scheduler via SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: Update scheduling application with the new or updated ask.
+ 3.2: rejections are send back to the RM via RMRejectedAllocationAskEvent
+ 3.3: accepted asks are not confirmed to RM or cache
+```
+
+**ask delete**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewAndReleaseAllocationRequests
+ 2.1: (by-pass): Send to scheduler via event SchedulerAllocationUpdatesEvent
+3. scheduler.processAllocationReleaseByAllocationKey
+ 3.1: Update scheduling application and remove the ask.
+```
+
+### Operations to add, update or remove nodes
+The RM (shim) sends a complex `UpdateRequest` as defined in the scheduler interface.
+This message is wrapped by the RM proxy and forwarded to the cache for processing.
+The RM can request a node to be added, updated or removed.
+
+**node add**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNewSchedulableNodes
+ 2.1: node sanity check (such as existence of partition/node)
+ 2.2: Add new nodes to the partition.
+ 2.3: notify scheduler of new node via SchedulerNodeEvent
+3. notify RM of node additions and rejections via RMNodeUpdateEvent
+ 3.1: notify the scheduler of allocations to recover via SchedulerAllocationUpdatesEvent
+4. scheduler.processAllocationUpdateEvent
+ 4.1: scheduler creates a new ask based on the Allocation to recover
+ 4.2: recover the allocation on the new node using a special process
+ 4.3: confirm the allocation in the scheduler, on failure update the cache with a ReleaseAllocationsEvent
+```
+
+**node update and removal**
+```
+1. RMProxy sends cacheevent.RMUpdateRequestEvent to cache
+2. cluster_info.processNodeActions
+ 2.1: node sanity check (such as existence of partition/node)
+ 2.2: Node info update (resource change)
+ 2.2.1: update node in cache
+ 2.2.2: notify scheduler of the node update via SchedulerNodeEvent
+ 2.3: Node status update (not removal), update node status in cache only
+ 2.4: Node removal
+ 2.4.1: update node status and remove node from the cache
+ 2.4.2: remove alloations and inform RM via RMReleaseAllocationEvent
+ 2.4.3: notify scheduler of the node removal via SchedulerNodeEvent
+3. scheduler.processNodeEvent add/remove/update the node
+```
+
+### Operations to add, update or remove partitions
+**Add RM**
+```
+1. RMProxy sends commonevents.RemoveRMPartitionsEvent
+ if RM is already registered
+ 1.1: scheduler.removePartitionsBelongToRM
+ 1.1.1: scheduler cleans up
+ 1.1.2: scheduler sends commonevents.RemoveRMPartitionsEvent
+ 1.2: cluster_info.processRemoveRMPartitionsEvent
+ 1.2.1: cache cleans up
+2. RMProxy sends commonevents.RegisterRMEvent
+3. cluster_info.processRMRegistrationEvent
+ 2.1: cache update internal partitions/queues accordingly.
+ 2.2: cache sends to scheduler SchedulerUpdatePartitionsConfigEvent.
+3. scheduler.processUpdatePartitionConfigsEvent
+ 3.1: Scheduler update partition/queue info accordingly.
+```
+
+**Update and Remove partition**
+Triggered by a configuration file update.
+```
+1. RMProxy sends commonevents.ConfigUpdateRMEvent
+2. cluster_info.processRMConfigUpdateEvent
+ 2.1: cache update internal partitions/queues accordingly.
+ 2.2: cache sends to scheduler SchedulerUpdatePartitionsConfigEvent.
+ 2.3: cache marks partitions for deletion (not removed yet).
+ 2.4: cache sends to scheduler SchedulerDeletePartitionsConfigEvent
+3. scheduler.processUpdatePartitionConfigsEvent
+ 3.1: scheduler updates internal partitions/queues accordingly.
+4. scheduler.processDeletePartitionConfigsEvent
+ 4.1: Scheduler set partitionManager.stop = true.
+ 4.2: PartitionManager removes queues, applications, nodes async.
+ This is the REAL CLEANUP including the cache
+```
+
+### Allocations
+Allocations are initiated by the scheduling process.
+The scheduler creates a SchedulingAllocation on the scheduler side which then gets wrapped in an AllocationProposal.
+The scheduler has checked resources etc already and marked the allocation as inflight.
+This description picks up at the point the allocation will be confirmed and finalised.
+
+**New allocation**
+```
+1. Scheduler wraps an SchedulingAllocation in an AllocationProposalBundleEvent
+2. cluster_info.processAllocationProposalEvent
+ preemption case: release preempted allocations
+ 2.1: release the allocation in the cache
+ 2.2: inform the scheduler the allocation is released via SchedulerNodeEvent
+ 2.3: inform the RM the allocation is released via RMReleaseAllocationEvent
+ all cases: add the new allocation
+ 2.4: add the new allocation to the cache
+ 2.5: rejections are send back to the scheduler via SchedulerAllocationUpdatesEvent
+ 2.6: inform the scheduler the allocation is added via SchedulerAllocationUpdatesEvent
+ 2.7: inform the RM the allocation is added via RMNewAllocationsEvent
+3. scheduler.processAllocationUpdateEvent
+ 3.1: confirmations are added to the scheduler and change from inflight to confirmed.
+ On failure of processing a ReleaseAllocationsEvent is send to the cache *again* to clean up.
+ This is part of the issue in [YUNIKORN-169]
+ cluster_info.processAllocationReleases
+ 3.2: rejections remove the inflight allocation from the scheduler.
+```
+
+## Current locking
+**Cluster Lock:**
+A cluster contains one or more Partition objects. A partition is a sub object of Cluster.
+Adding or Removing ANY Partition requires a write-lock of the cluster.
+Retrieving any object within the cluster will require iterating over the Partition list and thus a read-lock of the cluster
+
+**Partition Lock:**
+The partition object contains all links to Queue, Application or Node objects.
+Adding or Removing ANY Queue, Application or Node needs a write-lock of the partition.
+Retrieving any object within the partition will require a read-lock of the partition to prevent data races
+
+Examples of operation needing a write-lock
+- Allocation processing after scheduling, will change application, queue and node objects.
+ Partition lock is required due to possible updates to reservations.
+- Update of Node Resource
+ It not only affect node's available resource, it also affects the Partition's total allocatable Resource
+
+Example of operations that need a read-lock:
+- Retrieving any Queue, Application or Node needs a read-lock
+ The object itself is not locked as part of the retrieval
+- Confirming an allocation after processing in the cache
+ The partition is only locked for reading to allow retrieval of the objects that will be changed.
+ The changes are made on the underlying objects.
+
+Example of operations that do not need any lock:
+- Scheduling
+ Locks are taken on the specific objects when needed, no direct updates to the partition until the allocation is confirmed.
+
+**Queue lock:**
+A queue can track either applications (leaf type) or other queues (parent type).
+Resources are tracked for both types in the same way.
+
+Adding or removing an Application (leaf type), or a direct child queue (parent type) requires a write-lock of the queue.
+Updating tracked resources requires a write-lock.
+Changes are made recursively never locking more than 1 queue at a time.
+Updating any configuration property on the queue requires a write-lock.
+Retrieving any configuration value, or tracked resource, application or queue requires a read-lock.
+
+Examples of operation needing a write-lock
+- Adding an application to a leaf queue
+- Updating the reservations
+
+Examples of operation needing a read-lock
+- Retrieving an application from a leaf type queue
+- Retrieving the pending resources
+
+**Application lock:**
+An application tracks resources of different types, the allocations and outstanding requests.
+Updating any tracked resources, allocations or requests requires a write-lock.
+Retrieving any of those values requires a read-lock.
+
+Scheduling also requires a write-lock of the application.
+During scheduling the write-lock is held for the application.
+Locks will be taken on the node or queue that need to be accessed or updated.
+Examples of the locks taken on other objects are:
+- a read lock to access queue tracked resources
+- a write-lock to update the in progress allocations on the node
+
+Examples of operation needing a write-lock
+- Adding a new ask
+- Trying to schedule a pending request
+
+Examples of operation needing a read-lock
+- Retrieving the allocated resources
+- Retrieving the pending requests
+
+**Node lock:**
+An node tracks resources of different types and allocations.
+Updating any tracked resources or allocations requires a write-lock.
+Retrieving any of those values requires a read-lock.
+
+Checks run during the allocation phases take locks as required.
+Read-locks when checking write-locks when updating.
+A node is not locked for the whole allocation cycle.
+
+Examples of operation needing a write-lock
+- Adding a new allocation
+- updating the node resources
+
+Examples of operation needing a read-lock
+- Retrieving the allocated resources
+- Retrieving the reservation status
+
+## How to merge Cache and scheduler objects
+Since there is no longer the requirement to distinguish the objects in the cache and scheduler the `scheduling` and `info` parts of the name will be dropped.
+
+Overview of the main moves and merges:
+1. `application_info` & `scheduling_application`: **merge** to `scheduler.object.application`
+2. `allocation_info` & `scheduling_allocation`: **merge** to `scheduler.object.allocation`
+3. `node_info` & `scheduling_node`: **merge** to `scheduler.object.node`
+4. `queue_info` & `scheduling_queue`: **merge** to `scheduler.object.queue`
+5. `partition_info` & `scheduling_partition`: **merge** to `scheduler.PartitionContext`
+6. `cluster_info` & `scheduling_context`: **merge** to `scheduler.ClusterContext`
+7. `application_state`: **move** to `scheduler.object.applicationState`
+8. `object_state`: **move** to `scheduler.object.objectState`
+9. `initializer`: **merge** into `scheduler.ClusterContext`
+
+This move and merge of code includes a refactor of the objects into their own package.
+That thus affects the two scheduler only objects, reservations and schedulingAsk, that are already defined.
+Both will be moved into the objects package.
+
+The top level scheduler package remains for the contexts and scheduling code.
+
+## Code merges
+The first change is the event processing.
+All RM events will now directly be handled in the scheduler.
+Event handling will undergo a major change, far more than a simple merge.
+Only the RM generated events will be left after the merge.
+As described in the analysis above the scheduler is, in almost all cases, notified of changes from RM events.
+
+Broadly speaking there are only three types of changes triggered by the event removal:
+- configuration changes: new scheduler code required as the cache handling is not transferable to the scheduler
+- node, ask and application changes: merge of the cache code into the scheduler
+- allocation changes: removal of confirmation cycle and simplification of the scheduler code
+
+Part of the event handling is the processing of the configuration changes.
+All configuration changes will now update the scheduler objects directly.
+The way the scheduler works is slightly different from the cache which means the code is not transferable.
+
+Nodes and applications are really split between the cache and scheduler.
+Anything that is tracked in the cache object that does not have an equivalent value in the scheduler object will be moved into the scheduler object.
+All references to scheduler objects will be removed.
+With the code merges existing scheduler code that calls out directly into the cache objects will return the newly tracked value in the scheduler object.
+These calls will thus become locked calls in the scheduler.
+
+The concept of an in flight allocation will be removed.
+Allocation will be made in the same scheduling iteration without events or creation of a proposal.
+Removing the need for tracking of allocating resources on the scheduler objects.
+In flight resource tracking was required to make sure that an allocation while not confirmed by the cache would being taken into account while making scheduling decisions.
+
+The application and object state will be an integrated part of the scheduler object.
+A state change is thus immediate and this should prevent an issue like [YUNIKORN-169](https://issues.apache.org/jira/browse/YUNIKORN-169) from occuring.
+
+## Locking after merge
+
+### Direction of lock
+It is possible to acquire another lock while holding a lock, but we need to make sure that we do not allow:
+- Holding A.lock and acquire B's lock.
+- Holding B.lock and acquire B's lock.
+
+The current code in the scheduler takes a lock as late as possible and only for the time period needed.
+Some actions are not locked on the scheduler side just on the cache side as each object has its own lock.
+This means that a read of a value from the cache would not lock the scheduling object.
+
+With the integration of the cache into the scheduler the number of locks will decrease as the number of objects decreases.
+Each equivalent object, cache and scheduler, which used to have their own lock will now have just one.
+After the merge of the code is performed one lock will be left.
+Locking will occur more frequently as the number of fields in the scheduler objects has increased.
+
+Calls that did not lock the scheduler object before the merge will become locked.
+Lock contention could lead to performance degradation.
+The reduced overhead in objects and event handling can hopefully compensate for this.
+One point to keep track of is the change in locking behaviour.
+New behaviour could lead to new deadlock situations when code is simply merged without looking at the order.
+
+### Mitigations for deadlocks
+The locking inside the scheduler will be left as is.
+This means that the main scheduling logic will be taking and releasing locks as required on the objects.
+There are no long held read-locks or write-locks until the application is locked to schedule it.
+
+A major point of attention will need to be that no iterations of objects should be performed while holding on to a lock.
+For instance during scheduling while iterating over a queue's application we should not lock the queue.
+
+Another example would be that event processing in the partition should not lock the partition unneeded.
+The partition should be locked while retrieving for instance the node that needs updating and release the lock before it tries to lock the node itself.
+
+This approach fits in with the current locking approach and will keep the locking changes to a minimum.
+Testing, specifically end-to-end testing, should catch these deadlocks.
+There are no known tools that could be used to detect or describe lock order.
diff --git a/versioned_docs/version-1.5.0/design/config_v2.md b/versioned_docs/version-1.5.0/design/config_v2.md
new file mode 100644
index 0000000..e943bf4
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/config_v2.md
@@ -0,0 +1,346 @@
+---
+id: config_v2
+title: Configuration V2
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Over time the YuniKorn configuration has increased in complexity. The current
+configuration for YuniKorn is split over a number of different options, file
+and command line, and formats. There is a requirement now to define an overall
+configuration approach that can be used for all components.
+
+### Goals
+
+- Define a configuration format
+- Define an update and modification strategy
+- Provide consistency between configuration of YuniKorn components
+- Legacy migration strategy
+
+### Non-Goals
+
+- Redefine the queue configuration
+- Migration timeline for the old configuration options
+
+### References
+
+#### Implementation JIRA
+
+[YUNIKORN-1221](https://issues.apache.org/jira/browse/YUNIKORN-1221) - [Umbrella] Configuration V2
+
+#### Other relevant JIRAs
+
+- [YUNIKORN-455](https://issues.apache.org/jira/browse/YUNIKORN-455) - Make the core configurable
+- [YUNIKORN-1213](https://issues.apache.org/jira/browse/YUNIKORN-1213) - Background health checker needs to be configurable
+- [YUNIKORN-1335](https://issues.apache.org/jira/browse/YUNIKORN-1335) - Write yunikorn-web error logs to stderr
+
+### Services to cover
+
+- Scheduler (Standard) - K8Shim + Core
+- Scheduler (Plugin) - K8Shim + Core
+- Admission controller
+
+### Services excluded
+
+ - Web UI (not needed after [YUNIKORN-1335](https://issues.apache.org/jira/browse/YUNIKORN-1335))
+
+## Configuration format
+
+Currently only Kubernetes is a supported deployment pattern. Future shims may
+need an alternative configuration syntax. To enable this, we will update the
+scheduler interface to support passing the core configuration (the content of
+the `queues.yaml` ConfigMap entry) to the core on startup and refresh. The
+existing config watcher functionality in the core will be removed and the shim
+will gain the responsibility for watching for ConfigMap changes. The shim
+actually already has this ability now, it is just not used.
+
+We will use two ConfigMaps to configure YuniKorn:
+ - `yunikorn-defaults`
+ - This will be a new ConfigMap written by Helm; it will contain only values
+ explicitly configured by an administrator via Helm variables.
+ - `yunikorn-configs`
+ - This ConfigMap will contain overrides for yunikorn-defaults. It was
+ previously a Helm hook in previous YuniKorn releases; it will no longer
+ be managed by Helm and may be updated freely by third-party tools.
+
+The effective configuration will be the combination of both `yunikorn-defaults`
+ and `yunikorn-configs`, with `yunikorn-configs` taking precedence.
+
+Neither ConfigMap (even the existing one) is required for proper functionality.
+In the case where a ConfigMap does not exist, we will use compiled-in defaults
+for all options. This enables bootstrapping in a variety of environments,
+including local development mode, as the ConfigMap need not exist to bring the
+scheduler components up. Future configuration options should be set via the
+ConfigMap exclusively.
+
+### Mutability of configuration values
+
+We should strive for hot-reload of all configuration values where possible.
+Where hot-reload is not implemented, this should be clearly documented. For the
+initial release, the core config and admission controller will be
+hot-reloadable but many items in the shim will likely not be. The shim was not
+designed to hot-reload anything so this will be a large future task. Some items
+may never be reloadable due to architectural limitations.
+
+### Update handling
+
+Given the security issues involved with updates, the default Kubernetes access
+control alone should be used for ConfigMap updates. We can still syntax-check
+the core side of the configuration in the admission controller as before.
+Partial updates are allowed as Kubernetes supports patching of ConfigMaps.
+
+### Legacy configuration handling
+
+Helm will be updated to render all existing legacy options into the
+`yunikorn-defaults` ConfigMap for easier transition to the new configuration
+format. Consequently, it is not necessary for YuniKorn components to parse the
+old environment and command-line arguments.
+
+### Environment variables
+
+All environment variables for all services will be removed, with the exception
+of:
+
+ - `NAMESPACE`
+ - Default value: `default`
+ - Will be auto-populated from Helm and resolve to the pod's running
+ namespace
+ - Needed so that YuniKorn can determine what namespace it was launched in
+ - Should only be provided explicitly at development time
+ - `KUBECONFIG`
+ - Default value: `$HOME/.kube/config`
+ - Will be ignored if running inside Kubernetes; in-cluster configuration
+ will be used in that case
+ - Needed so that we can bootstrap to K8s and load the YuniKorn ConfigMap at
+ runtime
+ - Should only be provided explicitly at development time
+
+### Helm considerations
+
+All configuration options which have moved into the ConfigMap will be marked as
+deprecated in the Helm chart documentation. Their replacement is the more
+generic `yunikornDefaults` dictionary. If the legacy values are present but
+the associated `yunikornDefaults` entry is missing, the legacy configuration
+value will be used. This will allow users some time to migrate to the new
+configuration style. The legacy attributes will be removed in a future release.
+
+### Core Configuration
+
+The core scheduler configuration is loaded from the `queues.yaml` section of
+the ConfigMap, assuming `service.policyGroup` is set to the default value
+`queues`. If the policy group has been customized, core scheduler configuration
+will be read from `{service.policyGroup}.yaml`. This section is special as its
+contents are passed directly to the core for evaluation and update and is not
+read directly by the shim. The content of this section is unchanged from prior
+releases.
+
+## Appendix
+
+### Default ConfigMap
+
+If not present, the default ConfigMap will resolve to these defaults.
+Additionally, individual options will resolve to those listed here if not
+specified. Note that these values are subject to change in future releases.
+
+ apiVersion: v1
+ kind: ConfigMap
+ metadata:
+ name: yunikorn-configs
+ data:
+ service.clusterId: "mycluster"
+ service.policyGroup: "queues"
+ service.schedulingInterval: "1s"
+ service.volumeBindTimeout: "10s"
+ service.eventChannelCapacity: "1048576"
+ service.dispatchTimeout: "5m"
+ service.operatorPlugins: "general"
+ service.disableGangScheduling: "false"
+ service.enableConfigHotRefresh: "true"
+ service.placeholderImage: "registry.k8s.io/pause:3.7"
+ service.instanceTypeNodeLabelKey: "node.kubernetes.io/instance-type"
+ health.checkInterval: "30s"
+ log.level: "0"
+ kubernetes.qps: "1000"
+ kubernetes.burst: "1000"
+ admissionController.webHook.amServiceName: "yunikorn-admission-controller-service"
+ admissionController.webHook.schedulerServiceAddress: "yunikorn-service:9080"
+ admissionController.filtering.processNamespaces: ""
+ admissionController.filtering.bypassNamespaces: "^kube-system$"
+ admissionController.filtering.labelNamespaces: ""
+ admissionController.filtering.noLabelNamespaces: ""
+ admissionController.accessControl.bypassAuth: "false"
+ admissionController.accessControl.trustControllers: "true"
+ admissionController.accessControl.systemUsers: "^system:serviceaccount:kube-system:"
+ admissionController.accessControl.externalUsers: ""
+ admissionController.accessControl.externalGroups: ""
+ queues.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'`
+
+### Admission Controller Configuration
+
+#### Removed Environment Variables
+ - `POLICY_GROUP`
+ - Use ConfigMap: `service.policyGroup`
+ - `ENABLE_CONFIG_HOT_REFRESH`
+ - Use ConfigMap: `service.enableConfigHotRefresh`
+ - `SCHEDULER_SERVICE_ADDRESS`
+ - Use ConfigMap: `admissionController.webHook.schedulerServiceAddress`
+ - `ADMISSION_CONTROLLER_SERVICE`
+ - Use ConfigMap: `admissionController.webHook.amServiceName`
+ - `ADMISSION_CONTROLLER_NAMESPACE`
+ - Use Environment variable: `NAMESPACE`
+ - `ADMISSION_CONTROLLER_PROCESS_NAMESPACES`
+ - Use ConfigMap: `admissionController.filtering.processNamespaces`
+ - `ADMISSION_CONTROLLER_BYPASS_NAMESPACES`
+ - Use ConfigMap: `admissionController.filtering.bypassNamespaces`
+ - `ADMISSON_CONTROLLER_LABEL_NAMESPACES`
+ - Use ConfigMap: `admissionController.filtering.labelNamespaces`
+ - `ADMISSION_CONTROLLER_NO_LABEL_NAMESPACES`
+ - Use ConfigMap: `admissionController.filtering.noLabelNamespaces`
+
+#### Removed CLI Options
+None
+
+### Scheduler (Standard) Configuration
+
+#### Removed Environment Variables
+ - `USER_LABEL_KEY`
+ - No replacement
+ - `OPERATOR_PLUGINS`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `PLACEHOLDER_IMAGE`
+ - Use ConfigMap: `service.placeholderImage`
+
+#### Removed CLI Options
+ - `-clusterId`
+ - Use ConfigMap: `service.clusterId`
+ - `-clusterVersion`
+ - No replacement
+ - `-policyGroup`
+ - Use ConfigMap: `service.policyGroup`
+ - `-interval`
+ - Use ConfigMap: `service.schedulingInterval`
+ - `-logLevel`
+ - Use ConfigMap: `log.level`
+ - `-logEncoding`
+ - No replacement
+ - `-logFile`
+ - No replacement
+ - `-volumeBindTimeout`
+ - Use ConfigMap: `service.volumeBindTimeout`
+ - `-eventChannelCapacity`
+ - Use ConfigMap: `service.eventChannelCapacity`
+ - `-dispatchTimeout`
+ - Use ConfigMap: `service.dispatchTimeout`
+ - `-kubeQPS`
+ - Use ConfigMap: `kubernetes.qps`
+ - `-kubeBurst`
+ - Use ConfigMap: `kubernetes.burst`
+ - `-operatorPlugins`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `-enableConfigHotRefresh`
+ - Use ConfigMap: `service.enableConfigHotRefresh`
+ - `-disableGangScheduling`
+ - Use ConfigMap: `service.disableGangScheduling`
+ - `-userLabelKey`
+ - No replacement
+ - `-placeHolderImage`
+ Use ConfigMap: `service.placeholderImage`
+
+### Scheduler (Plugin) Configuration
+
+#### Removed Environment Variables
+ - `USER_LABEL_KEY`
+ - No replacement
+ - `OPERATOR_PLUGINS`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `PLACEHOLDER_IMAGE`
+ - Use ConfigMap: `service.placeholderImage`
+
+#### Removed CLI Options
+ - `--yk-scheduler-name`
+ - No replacement
+ - `--yk-cluster-id`
+ - Use ConfigMap: `service.clusterId`
+ - `--yk-cluster-version`
+ - No replacement
+ - `--yk-policy-group`
+ - Use ConfigMap: `service.policyGroup`
+ - `--yk-scheduling-interval
+ - Use ConfigMap: `service.schedulingInterval`
+ - `--yk-kube-config`
+ - Use environment: `KUBECONFIG`
+ - `--yk-log-level`
+ - Use ConfigMap: `log.level`
+ - `--yk-log-encoding`
+ - No replacement
+ - `--yk-log-file`
+ - No replacement
+ - `--yk-event-channel-capacity`
+ - Use ConfigMap: `service.eventChannelCapacity`
+ - `--yk-dispatcher-timeout`
+ - Use ConfigMap: `service.dispatchTimeout`
+ - `--yk-kube-qps`
+ - Use ConfigMap: `kubernetes.qps`
+ - `--yk-kube-burst`
+ - Use ConfigMap: `kubernetes.burst`
+ - `--yk-operator-plugins`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `--yk-enable-config-hot-refresh`
+ - Use ConfigMap: `service.enableConfigHotRefresh`
+ - `--yk-disable-gang-scheduling`
+ - Use ConfigMap: `service.disableGangScheduling`
+ - `--yk-user-label-key`
+ - No replacement
+ - `--yk-placeholder-image`
+ - Use ConfigMap: `service.placeholderImage`
+
+### Helm Configuration
+
+#### Removed Options
+ - `userLabelKey`
+ - No replacement
+
+#### Deprecated Options
+ - `operatorPlugins`
+ - Use ConfigMap: `service.operatorPlugins`
+ - `placeHolderImage`
+ - Use ConfigMap: `service.placeholderImage`
+ - `admissionController.processNamespaces`
+ - Use ConfigMap: `admissionController.filtering.processNamespaces`
+ - `admissionController.bypassNamespaces`
+ - Use ConfigMap: `admissionController.filtering.bypassNamespaces`
+ - `admissionController.labelNamespaces`
+ - Use ConfigMap: `admissionController.filtering.labelNamespaces`
+ - `admissionController.noLabelNamespaces`
+ - Use ConfigMap: `admissionController.filtering.noLabelNamespaces`
+ - `configuration`
+ - Use ConfigMap: `queues.yaml`
+
diff --git a/versioned_docs/version-1.5.0/design/gang_scheduling.md b/versioned_docs/version-1.5.0/design/gang_scheduling.md
new file mode 100644
index 0000000..44560ef
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/gang_scheduling.md
@@ -0,0 +1,605 @@
+---
+id: gang_scheduling
+title: Gang scheduling design
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+# Gang Scheduling Implementation
+A new way of scheduling applications by taking into account the demand for resources the application expects it will generate over time.
+It guarantees the expected demand resources for the application by reserving the resources.
+
+There are two parts to this implementation:
+* Kubernetes Shim
+* Core and scheduling
+
+This document describes the implementation on the core side.
+
+## Document goals
+This document describes the following implementation design points:
+1. Define changes required for the shim to core communication (scheduler interface)
+2. Scheduler storage object changes
+3. Scheduler logic changes
+
+## Excluded design points
+Currently, the Kubernetes shim side implementation is not covered in this design document.
+
+Generalised preemption on the core side will _not_ be discussed in this design.
+
+## Generic flow
+The flow is triggered by a pod that is submitted which triggers the application creation.
+This first pod is in the case of a Spark application, the driver pod.
+In case the flow is triggered from the creation of an application CRD there will not be a first pod.
+This is however outside the core scheduling logic. From the core side there should be no difference between the two cases.
+More details are in the chapter on the [Scheduler logic changes](#scheduler-logic-changes).
+
+The flow of an application submitted. The numbers in the diagram correspond to the description below the diagram.
+
+
+
+Combined flow for the shim and core during startup of an application:
+* An application is submitted with TaskGroup(s) defined. (1)
+* The shim creates the application and passes the application to the core. (2)
+* The shim creates placeholder pods for each of the members of the TaskGroup(s) (3)
+* The pods are processed and passed to the core, as per the normal behaviour, as AllocationAsks for the application with the correct info set. (4)
+* The placeholder AllocationAsk’s are scheduled by the core as if they were normal AllocationAsk’s. (5)
+* All Allocations, even if they are the result of the placeholder AllocationAsks being allocated by the scheduler, are communicated back to the shim.
+* The original real pod is passed to the core as an AllocationAsk. (6)
+* After the real pod and all all the placeholder pods are scheduled the shim starts the real pod that triggered the application creation. (7)
+
+After the first, real, pod is started the following pods should all be handled in the same way (8):
+* A real pod is created on k8s.
+* The pod is processed and an AllocationAsk is created.
+* The scheduler processes the AllocationAsk (more detail below) and replaces a placeholder with the real allocation.
+
+## Application submit handling
+### Total placeholder size
+
+The application if requesting one or more TaskGroups should provide the total size of all the TaskGroup members it is going to request.
+The total resource size is required for the case that the application is scheduled in a queue with a resource limit set.
+
+The value is important for three cases:
+1. gang request is larger than the queue quota
+2. start of scheduling reservations
+3. resource pressure while scheduling reservations
+
+Further detail will be given below in [scheduling in queues with a quota set](#scheduling-in-queues-with-a-quota-set)
+
+The information passed on from the shim should be part of the AddApplicationRequest.
+Detailed information on the build up of the taskGroup(s), or the number of members are not relevant.
+The total resource requested by all taskGroup members is calculated using:
+
+
+
+This total placeholderAsk is added as an optional field to the AddApplicationRequest message.
+The calculation can be made by the shim based on the CRD or annotation provided in the pod description.
+
+If the placeholderAsk is larger than the queue quota set on the queue the application must be rejected.
+This rejection is based on the fact that we cannot in any way honor the request.
+For all other cases the application is accepted and will be scheduled as per normal.
+
+### Handling queue with a FAIR sort policy
+If an application is submitted to a queue that has a FAIR sort policy set it must be rejected.
+Queue sorting for the queue that the application runs in must be set to _FIFO_ or _StateAware_.
+
+Other queue policies cannot guarantee that there is only one _New_ application processed at a time.
+In the case of the _FAIR_ policy we could be allocating multiple _New_ applications at the same time making quota management impossible to enforce.
+The other side effect of using _FAIR_ as a policy could be that we get multiple applications with only a partial allocated guarantee.
+
+Auto-scaling can be triggered due to the fact that the core can not place the placeholders on any node.
+In case the queue would use the _FAIR_ sorting this could lead to other applications taking the scaled up nodes instead of the placeholders again breaking the gang.
+
+## Scheduling in queues with a quota set
+The main case already described above is handling a total placeholder request size that is larger than the quota set on the queue.
+When the application is submitted we can already assess that we cannot satisfy that requirement and reject the request.
+
+In the case that the total placeholder ask does fit in the queue we should not start scheduling until there are enough resources available in the queue to satisfy the total request.
+However this does not stop scheduling of other applications in the queue(s).
+Applications that are already running in the queue could ask for more resources.
+From an application perspective there is no limit set on the resource it can request.
+The gang defined on the application is a guaranteed number of resources, not a maximum number of resources the application can request.
+
+This is complicated by the fact that we have a queue hierarchy.
+There is the possibility that the quota is not set directly on the queue the application is running.
+It could be set on one of the parent queues.
+This case could become complex, and we need to make sure that we keep in mind that we could live lock the scheduling.
+
+In this first phase we should focus on the case that the gang resources requested are also the maximum number of resources the application will request.
+When we look at the queues we should focus on a single queue level with quotas.
+
+These two assumptions are correct for the spark use case without dynamic allocation using a dynamic mapping from a namespace to a queue.
+
+Furthermore, we assume that the quota set on the queue can be totally allocated.
+If the cluster does not have enough resources the cluster will scale up to the size needed to provide all queues with their full quota.
+
+The follow up should add further enhancements for deeper hierarchies and dynamic allocation support.
+This could also leverage preemption in certain use cases, like preempting allocations from applications over their total gang size.
+
+Further enhancements could be added by allowing specifying the time and application will wait for the placeholders to be allocated, or the time to start using the held resources.
+
+## Scheduler logic changes
+The scheduler logic change needs to account for two parts of cycle:
+* The placeholder asks and their allocation.
+* The allocation replacing the placeholder.
+
+The basic assumption is that all pods will generate a placeholder pod request to the core.
+This includes the pod that triggered the application creation if we do not use the application CRD.
+This assumption is needed to make sure that the scheduler core can behave in the same way for both ways of submitting the application.
+The placeholder pods must be communicated to the core before the real pod.
+
+Changes for the placeholder AllocationAsks are the first step.
+As part of the creation of the application the AllocationAsks get added.
+The addition of an AllocationsAsk normally will trigger the application state change as per the scheduling cycle.
+It moves the Application from a _New_ state to an _Accepted_ state. This is as per the current setup, and does not change.
+
+However, in the case that the AllocationAsk has the _placeholder_ flag set the allocation should not trigger a state change, the application stays in _Accepted_ state.
+AllocationAsks are processed until the application has no pending resources.
+AllocationAsks that do not have a _placeholder_ flag set should be ignored as a safety precaution.
+All resulting Allocations for the placeholder pods are confirmed to the shim as per the normal steps.
+This process continues until there are no more placeholder pods to be allocated.
+
+The shim at that point should create the AllocationAsk for the real pod(s) that it has buffered.
+The core cannot and must not assume that there is only one task group per application.
+The core is also not in the position to assume that it has received all AllocationAsks that belong to the task group if option 1 as described above is used by a shim.
+This is also why we have the assumption that every pod creates a placeholder request to the core.
+
+The second change is the replacement of the placeholder pods with the real pods.
+The shim creates an AllocationAsk with the _taskGroupName_ set but the _placeholder_ flag is not set.
+
+The process described here lines up with the process for generic pre-emption.
+An allocation is released by the core and then confirmed by the shim.
+For gang scheduling we have a simple one new to one release relation in the case of pre-emption we can use the same flow with a one new to multiple release relation.
+
+The scheduler processes the AllocationAsk as follows:
+1. Check if the application has an unreleased allocation for a placeholder allocation with the same _taskGroupName._ If no placeholder allocations are found a normal allocation cycle will be used to allocate the request.
+2. A placeholder allocation is selected and marked for release. A request to release the placeholder allocation is communicated to the shim. This must be an async process as the shim release process is dependent on the underlying K8s response which might not be instantaneous.
+ NOTE: no allocations are released in the core at this point in time.
+3. The core “parks” the processing of the real AllocationAsk until the shim has responded with a confirmation that the placeholder allocation has been released.
+ NOTE: locks are released to allow scheduling to continue
+4. After the confirmation of the release is received from the shim the “parked” AllocationAsk processing is finalised.
+5. The AllocationAsk is allocated on the same node as the placeholder used.
+ The removal of the placeholder allocation is finalised in either case. This all needs to happen as one update to the application, queue and node.
+ * On success: a new Allocation is created.
+ * On Failure: try to allocate on a different node, if that fails the AllocationAsk becomes unschedulable triggering scale up.
+6. Communicate the allocation back to the shim (if applicable, based on step 5)
+
+## Application completion
+Application completion has been a long standing issue.
+Currently, applications do not transition to a _completed_ state when done.
+The current states for the application are [documented here](developer_guide/scheduler_object_states.md).
+However, at this point in time an application will not reach the _completed_ state and will be stuck in _waiting_.
+
+This provides a number of issues specifically around memory usage and cleanup of queues in long running deployments.
+
+### Definition
+Since we cannot rely on the application, running as pods on Kubernetes, to show that it has finished we need to define when we consider an application _completed_.
+At this point we are defining that an application is _completed_ when it has been in the _waiting_ state for a defined time period.
+An application enters the waiting state at the time that there are no active allocations (allocated resources > 0) and pending allocation asks (pending resources > 0).
+
+The transition to a _waiting_ state is already implemented.
+The time out of the _waiting_ state is new functionality.
+
+Placeholders are not considered active allocations.
+Placeholder asks are considered pending resource asks.
+These cases will be handled in the [Cleanup](#Cleanup) below.
+
+### Cleanup
+When we look at gang scheduling there is a further issue around unused placeholders, placeholder asks and their cleanup.
+Placeholders could be converted into real allocations at any time there are pending allocation asks or active allocations.
+
+Placeholder asks will all be converted into placeholder allocations before the real allocations are processed.
+
+Entry into the _waiting_ state is already handled.
+If new allocation asks are added to the application it will transition back to a _running_ state.
+At the time we entered the waiting state. there were no pending requests or allocated resources.
+There could be allocated placeholders.
+
+For the entry into the _waiting_ state the application must be clean.
+However, we can not guarantee that all placeholders will be used by the application during the time the application runs.
+Transitioning out of the _waiting_ state into the _completed_ state requires no (placeholder) allocations or asks at all.
+The second case that impact transitions is that not all placeholder asks are allocated, and the application thus never requests any real allocations.
+These two cases could prevent an application from transitioning out of the _accepted_, or the _waiting_ state.
+
+Processing in the core thus needs to consider two cases that will impact the transition out of specific states:
+1. Placeholder asks pending (exit from _accepted_)
+2. Placeholders allocated (exit from _waiting_)
+
+Placeholder asks pending:
+Pending placeholder asks are handled via a timeout.
+An application must only spend a limited time waiting for all placeholders to be allocated.
+This timeout is needed because an application’s partial placeholders allocation may occupy cluster resources without really using them.
+
+An application could be queued for an unknown time, waiting for placeholder allocation to start.
+The timeout for placeholder asks can thus not be linked to the creation of the application or the asks.
+The timeout must start at the time the first placeholder ask is allocated.
+
+The application cannot request real allocations until all placeholder asks are allocated.
+A placeholder ask is also tracked by the shim as it represents a pod.
+Releasing an ask in the core requires a message to flow between the core and shim to release that ask.
+However, in this case the timeout for allocating placeholder asks triggers an application failure.
+When the timeout is triggered and placeholder asks are pending the application will transition from the state it is in, which can only be _accepted_, to _killed_.
+
+The application state for this case can be summarised as:
+* Application status is _accepted_
+* Placeholder allocated resource is larger than zero, and less than the _placeholderAsk_ from the _AddApplicationRequest_
+* Pending resource asks is larger than zero
+
+Entering into the _killed_ state must move the application out of the queue automatically.
+
+The state change and placeholder allocation releases can be handled in a single UpdateResponse message. The message will have the following content:
+* _UpdatedApplication_ for the state change of the application
+* one or more _AllocationRelease_ messages, one for each placeholder, with the _TerminationType_ set to TIMEOUT
+* one or more AllocationAskRelease messages with the _TerminationType_ set to TIMEOUT
+
+The shim processes the AllocationAskRelease messages first, followed by the _AllocationResponse_ messages, and finally the _UpdatedApplication_ message. The application state change to the _killed_ state on the core side is only dependent on the removal of all placeholders pods, not on a response to the _UpdatedApplication_ message.
+
+
+
+Combined flow for the shim and core during timeout of placeholder:
+* The core times out the placeholder allocation. (1)
+* The placeholder Allocations removal is passed to the shim. (2)
+* All placeholder Allocations are released by the shim, and communicated back to the core.
+* The placeholder AllocationAsks removal is passed to the shim. (3)
+* All placeholder AllocationAsks are released by the shim, and communicated back to the core.
+* After the placeholder Allocations and Asks are released the core moves the application to the killed state removing it from the queue (4).
+* The state change is finalised in the core and shim. (5)
+
+Allocated placeholders:
+Leftover placeholders need to be released by the core.
+The shim needs to be informed to remove them. This must be triggered on entry of the _completed_ state.
+After the placeholder release is requested by the core the state transition of the application can proceed.
+The core will process the _AllocationRelease_ messages for placeholder allocations that come back from the shim with the _TerminationType_ set to TIMEOUT as normal without triggering a state change.
+
+The state change and placeholder allocation releases can be handled in a single UpdateResponse message.
+The message will have the following content:
+* _UpdatedApplication_ for the state change of the application
+* zero or more _AllocationRelease_ messages, one for each placeholder, with the _TerminationType_ set to TIMEOUT
+
+The shim processes the _AllocationResponse_ messages first followed by the _UpdatedApplication_ message.
+The application state change to the _completed_ state on the core side is only dependent on the removal of all placeholders pods, not on a response to the _UpdatedApplication_ message.
+
+Entering into the _completed_ state will move the application out of the queue automatically.
+This should also handle the case we discussed earlier around a possible delayed processing of requests from the shim as we can move back from _waiting_ to _running_ if needed.
+A _completed_ application should also not prevent the case that was discussed around cron like submissions using the same application ID for each invocation.
+A _completed_ application with the same application ID must not prevent the submission of a new application with the same ID.
+
+
+
+Combined flow for the shim and core during cleanup of an application:
+* A pod is released at the Kubernetes layer. (1)
+* The shim passes the release of the allocation on to the core. (2)
+* The core transitions the application to a waiting state if no pending or active allocations. (3)
+* The waiting state times out and triggers the cleanup. (4)
+* The placeholder Allocations removal is passed to the shim. (5)
+* All placeholder Allocations are released by the shim, and communicated back to the core.
+* After all placeholders are released the core moves the application to the completed state removing it from the queue (6).
+* The state change is finalised in the core and shim. (7)
+
+## Application recovery
+During application recovery the placeholder pods are recovered as any other pod on a node.
+These pods are communicated to the core by the shim as part of the node as an existing allocation.
+Existing allocations do not have a corresponding _AllocationAsk_ in the core. The core generates an _AllocationAsk_ based on the recovered information.
+
+For gang scheduling the _AllocationAsk_ contains the _taskGroupName_ and _placeholder_ flag.
+During recovery that same information must be part of the _Allocation_ message.
+This is due to the fact that the same message is used in two directions, from the RM to the scheduler and vice versa means we need to update the message and its processing.
+
+If the information is missing from the _Allocation_ message the recovered allocation will not be correctly tagged in the core.
+The recovered allocation will be seen as a regular allocation.
+This means it is skipped as part of the normal allocation cycle that replaces the placeholders.
+
+The logic change only requires that the recovery of existing allocations copies the fields from the interface message into the allocation object in the core.
+
+## Interface changes
+Multiple changes are needed to the communication between the shim and the core to support the gang information needed.
+
+An application must provide the total size of the placeholder requests to prevent accepting an application that can never run.
+
+The current object that is sent from the shim to the core for allocation requests is defined in the AllocationAsk.
+The Allocation, as the result message passed back from the scheduler core does not change. For recovery, which uses the same Allocation message, from the shim to the core, however must contain the gang related fields.
+Gang related fields must be added to both messages.
+
+The allocation release request and response request need to support bidirectional traffic and will need to undergo major changes.
+
+### AddApplication
+The AddApplicationRequest message requires a new field to communicate the total placeholder resource request that will be requested.
+The field is used to reject the application if it is impossible to satisfy the request.
+It can also be used to stop the core from scheduling any real pods for that application until all placeholder pods are processed.
+
+In patched message form that would look like:
+```
+message AddApplicationRequest {
+...
+ // The total amount of resources gang placeholders will request
+ Resource placeholderAsk = 7;
+...
+}
+```
+
+### AllocationAsk
+The first part of the change is the base information for the task group.
+This will require an additional optional attribute to be added.
+The content of this optional attribute is a name, a string, which will be mapped to the name of the task group.
+The field can be present on a real allocation and on a placeholder.
+
+Proposed name for the new field is: _taskGroupName_
+
+To distinguish normal AllocationAsks and placeholder AllocationAsks a flag must be added.
+The flag will never have more than two values and thus maps to a boolean. As the default value for a boolean is _false_ the field should show the fact that it is an AllocationAsk that represents a placeholder as true.
+
+Proposed name for the field is: _placeholder_
+
+In patched message form that would look like:
+```
+message AllocationAsk {
+...
+ // The name of the TaskGroup this ask belongs to
+ string taskGroupName = 10;
+ // Is this a placeholder ask (true) or a real ask (false), defaults to false
+ // ignored if the taskGroupName is not set
+ bool placeholder = 11;
+...
+}
+```
+
+The last part of the task group information that needs to be communicated is the size of the task group.
+This does not require a change in the interface as the current AllocationAsk object can support both possible options.
+
+Requests can be handled in two ways:
+1. Each member of the task group is passed to the core as a separate AllocationAsk with a maxAllocations, or the ask repeat, of 1
+2. The task group is considered one AllocationAsk with a maxAllocations set to the same value as minMember of the task group information.
+
+With option A the shim will need to generate multiple AllocationAsk objects and pass each to the core for scheduling.
+Each AllocationAsk is linked to one pod.
+Option B will only generate one AllocationAsk for all placeholder pods.
+Option B requires less code and has less overhead on the core side.
+However the logic on the shim side might be more complex as the returned allocation needs to be linked to just one pod.
+
+Proposal is to use option: A
+
+### Allocation
+Similar to the change for the _AllocationAsk_ the _Allocation_ requires additional optional attributes to be added.
+The new fields distinguish a normal Allocation and placeholder Allocations on recovery.
+The same rules apply to these fields as the ones added to the _AllocationAsk_.
+
+The content of this optional attribute is a name, a string, which will be mapped to the name of the task group.
+The field can be present on a real allocation and on a placeholder.
+
+Proposed name for the new field is: _taskGroupName_
+
+The flag will never have more than two values and thus maps to a boolean.
+As the default value for a boolean is _false_ the field should show the fact that it is an Allocation that represents a placeholder as true.
+
+Proposed name for the field is: _placeholder_
+
+In patched message form that would look like:
+```
+message Allocation {
+...
+ // The name of the TaskGroup this allocation belongs to
+ string taskGroupName = 11;
+ // Is this a placeholder allocation (true) or a real allocation (false), defaults to false
+ // ignored if the taskGroupName is not set
+ bool placeholder = 12;
+...
+}
+```
+
+### AllocationRelease Response and Request
+The name for the messages are based on the fact that the release is always triggered by the shim.
+In case of preemption and or gang scheduling the release is not triggered from the shim but from the core.
+That means the message name does not cover the usage. A response message might not have an associated request message.
+It could be used to indicate direction but that is in this case confusing.
+
+When a release is triggered from the core, for preemption or the placeholder allocation, a response is expected from the shim to confirm that the release has been processed.
+This response must be distinguished from a request to release the allocation initiated by the shim.
+A release initiated by the shim must be followed by a confirmation from the core to the shim that the message is processed.
+For releases initiated by the core no such confirmation message can or must be sent.
+In the current request message there is no way to indicate that it is a confirmation message.
+
+To fix the possible confusing naming the proposal is to merge the two messages into one message: _AllocationRelease_.
+
+The _AllocationReleaseRequest_ is indirectly part of the _UpdateRequest_ message as it is contained in the _AllocationReleasesRequest_.
+The _AllocationReleaseResponse_ is part of the _UpdateResponse_ message.
+The flow-on effect of the rename and merge of the two messages is a change in the two messages that contain them.
+The message changes for _UpdateResponse_ and _AllocationReleasesRequest_ are limited to type changes of the existing fields.
+
+| Message | Field ID | Old type | New type |
+| ------------------------- | -------- | ------------------------- | ----------------- |
+| UpdateResponse | 3 | AllocationReleaseResponse | AllocationRelease |
+| AllocationReleasesRequest | 1 | AllocationReleaseRequest | AllocationRelease |
+
+In patched message form that would look like:
+```
+message UpdateResponse {
+...
+ // Released allocation(s), allocations can be released by either the RM or scheduler.
+ // The TerminationType defines which side needs to act and process the message.
+ repeated AllocationRelease releasedAllocations = 3;
+...
+}
+
+message AllocationReleasesRequest {
+ // Released allocation(s), allocations can be released by either the RM or scheduler.
+ // The TerminationType defines which side needs to act and process the message.
+ repeated AllocationRelease releasedAllocations = 1;
+...
+}
+```
+
+The merged message _AllocationRelease_ will consist of:
+
+| Field name | Content type | Required |
+| --------------- | ----------------- | -------- |
+| partitionName | string | yes |
+| applicationID | string | no |
+| UUID | string | no |
+| terminationType | _TerminationType_ | yes |
+| message | string | no |
+
+Confirmation behaviour of the action should be triggered on the type of termination received.
+The core will confirm the release to the shim of all types that originate in the shim and vice versa.
+
+A confirmation or response uses the same _TerminationType_ as was set in the original message.
+An example of this is a pod that is removed from K8s will trigger an _AllocationRelease_ message to be sent from the shim to the core with the TerminationType STOPPED_BY_RM. The core processes the request removing the allocation from the internal structures, and when all processing is done it responds to the shim with a message using the same _TerminationType_.
+The shim can ignore that or make follow up changes if needed.
+
+A similar process happens for a release that originates in the core.
+Example of the core sending an _AllocationRelease_ message to the shim using the _TerminationType_ PREEMPTED_BY_SCHEDULER.
+The shim handles that by releasing the pod identified and responds to the core that it has released the pod.
+On receiving the confirmation that the pod has been released the core can progress with the allocation and preemption processing.
+
+In patched message form that would look like:
+```
+message AllocationRelease {
+ enum TerminationType {
+ STOPPED_BY_RM = 0;
+ TIMEOUT = 1;
+ PREEMPTED_BY_SCHEDULER = 2;
+ PLACEHOLDER_REPLACED = 3;
+ }
+
+ // The name of the partition the allocation belongs to
+ string partitionName = 1;
+ // The application the allocation belongs to
+ string applicationID = 2;
+ // The UUID of the allocation to release, if not set all allocations are released for
+ // the applicationID
+ string UUID = 3;
+ // The termination type as described above
+ TerminationType terminationType = 4;
+ // human-readable message
+ string message = 5;
+}
+```
+### TerminationType
+The currently defined _TerminationType_ values and specification of the side that generates (Sender), and the side that actions and confirms processing (Receiver):
+
+| Value | Sender | Receiver |
+| ------------------------ | ------ | -------- |
+| STOPPED_BY_RM | shim | core |
+| TIMEOUT * | core | shim |
+| PREEMPTED_BY_SCHEDULER * | core | shim |
+
+* currently not handled by the shim, core or both
+
+When the placeholder allocation gets released the _AllocationReleaseResponse_ is used to communicate the release back from the core to the shim.
+The response contains an enumeration called _TerminationType_, and a human-readable message.
+For tracking and tracing purposes we should add a new _TerminationType_ specifically for the placeholder replacement. The shim must take action based on the type and confirm the allocation release to the core.
+
+It should provide enough detail so we do not have to re-use an already existing type, or the human-readable message.
+The human-readable format can still be used to provide further detail on which new allocation replaced the placeholder.
+
+Proposal is to add: _PLACEHOLDER_REPLACED_
+
+| Value | Sender | Receiver |
+| ---------------------| ------ | -------- |
+| PLACEHOLDER_REPLACED | shim | core |
+
+As part of the Scheduler Interface cleanup ([YUNIKORN-486](https://issues.apache.org/jira/browse/YUNIKORN-486)) the _TerminationType_ should be extracted from the _AllocationRelease_ and _AllocationaskRelease_ message.
+It is an enumeration that can be shared between multiple objects.
+[YUNIKORN-547](https://issues.apache.org/jira/browse/YUNIKORN-547) has been logged to handle this as it has an impact on the code outside of the scope of gang scheduling.
+
+
+### AllocationAskRelease Response and Request
+The allocation ask release right now can only be triggered by the shim.
+In order for the core to perform the cleanup when the placeholder allocation times out, we need to make this a bidirectional message.
+Similarly to the Allocation we would rename the _AllocationAskReleaseRequest_ to _AllocationAskRelease_, so we can use this message in both directions:
+```
+message AllocationReleasesRequest {
+...
+ // Released allocationask(s), allocationasks can be released by either the RM or
+ // scheduler. The TerminationType defines which side needs to act and process the
+ // message.
+ repeated AllocationAskRelease allocationAsksToRelease = 2;
+}
+```
+
+Similar processing logic based on the _TerminationType_ which is used for allocations should be used for ask releases.
+In patched message form that would look like:
+```
+message AllocationAskRelease {
+ enum TerminationType {
+ STOPPED_BY_RM = 0;
+ TIMEOUT = 1;
+ PREEMPTED_BY_SCHEDULER = 2;
+ PLACEHOLDER_REPLACED = 3;
+ }
+...
+ // The termination type as described above
+ TerminationType terminationType = 4;
+...
+}
+```
+
+Confirmation behaviour of the action should be triggered on the type of termination received.
+The core will confirm the release to the shim of all types that originate in the shim and vice versa.
+
+A confirmation or response uses the same _TerminationType_ as was set in the original message.
+
+## Scheduler storage object changes
+### AllocationAsk
+In line with the changes for the communication the objects in the scheduler also need to be modified to persist some of the detail communicated.
+The AllocationAsk that is used in the communication has an equivalent object inside the scheduler with the same name.
+This object needs to be able to store the new fields proposed above.
+
+Proposed new fields: _taskGroupName_ and _placeholder_.
+
+In the current interface specification a field called _executionTimeoutMilliSeconds_ is defined.
+This is currently not mapped to the object inside the scheduler and should be added.
+Time or Duration are stored as native go objects and do not include a size specifier.
+
+Proposed new field: _execTimeout_
+
+### Allocation
+After the allocation is made an Allocation object is created in the core to track the real allocation. This Allocation object is directly linked to the application and should show that the allocation is a placeholder and for which task group. This detail is needed to also enable the correct display of the resources used in the web UI.
+
+The propagation of the placeholder information could be achieved indirectly as the allocation object references an AllocationAsk. This would require a lookup of the AllocationAsk to assess the type of allocation. We could also opt to propagate the data into the Allocation object itself. This would remove the lookup and allow us to directly filter allocations based on the type and or task group information.
+
+From a scheduling and scheduler logic perspective the indirect reference is not really desirable due to the overhead of the lookups required. This means that the same fields added in the AllocationAsk are also added to the Allocation object.
+
+Proposed new fields: _taskGroupName_ and _placeholder_.
+
+To support the release of the allocation being triggered from the core tracking of the release action is required. The release is not final until the shim has confirmed that release. However during that time period the allocation may not be released again.
+
+Proposed new field: _released_
+
+At the point that we replace the placeholder with a real allocation we need to release an existing placeholder.
+The Allocation object allows us to specify a list of Allocations to release.
+This field was added earlier to support preemption.
+This same field will be reused for the placeholder release.
+
+### Application
+The AddApplicationRequest has a new field added that needs to be persisted in the object inside the scheduler.
+
+Proposed new field: _placeholderAsk_
+
+In the current interface specification a field called _executionTimeoutMilliSeconds_ is defined. This is currently not mapped to the object inside the scheduler and should be added. Time or Duration are stored as native go objects and do not include a size specifier.
+
+Proposed new field: _execTimeout_
+
+The application object should be able to track the placeholder allocations separately from the real allocations. The split of the allocation types on the application will allow us to show the proper state in the web UI.
+
+Proposed new field: _allocatedPlaceholder_
+
+
+### Queue & Node
+No changes at this point.
+The placeholder allocations should be counted as “real” allocations on the Queue and Node.
+By counting the placeholder as normal the quota for the queue is enforced as expected.
+The Node object needs to also show normal usage to prevent interactions with the autoscaler.
diff --git a/versioned_docs/version-1.5.0/design/generic_resource.md b/versioned_docs/version-1.5.0/design/generic_resource.md
new file mode 100644
index 0000000..f4a8ce4
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/generic_resource.md
@@ -0,0 +1,69 @@
+---
+id: generic_resource
+title: Generic Resource Types in Namespace Quota
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+# Generic Resource Types in Namespace Quota
+Tracking jira: [YUNIKORN-1275](https://issues.apache.org/jira/browse/YUNIKORN-1279)
+
+## Functional gap
+The queue configuration allows all resource types to be set in a quota. Namespace annotations do not. Support for the same resource types in the annotations on namespaces should be possible.
+
+## Current solution
+In the current setup YuniKorn supports annotations on a namespace to specify a resource quota for that namespace. This is used in combination with placement rules to create a quota limited queue automatically based on the namespace in Kubernetes.
+
+The annotations that are supported limit the possible resource types that are supported on these auto created queues. Each resource type uses its own annotation. Current annotations supported as per the quota management documentation:
+```
+yunikorn.apache.org/namespace.max.cpu
+yunikorn.apache.org/namespace.max.memory
+```
+The queue configuration itself, as part of the yaml file, supports all Kubernetes resources including extended resources.
+## Proposed solution
+The current solution uses a specific annotation for each type that is supported. This means that each new resource would require a new annotation to be defined. Reading a new annotation requires a code change in the k8shim.
+
+In comparison when we look at the gang scheduling setup with the task groups specification we are far more flexible. In that case we allow a map of resources to be specified. The map uses the resource name as the key and allows a value as per the Kubenetes resource specification. This solution allows any resource type to be set as a request for a task group.
+
+An equivalent solution should be allowed for the quota annotation on the namespace. This would provide a more flexible solution that does not require code changes for every new resource type that must be supported as part of the namespace quota.
+
+### Annotation name
+The new name for the annotation should not interfere with the existing annotations that are used for the memory and cpu resource quota. Beside that rule we are free to use any name that complies with the naming conventions for names.
+
+The proposal is to use:
+```
+yunikorn.apache.org/namespace.quota
+```
+### Annotation content
+The content of the annotation must be a simple string. There are no length limits for a specific annotation. All annotations together on one object do have a size limit however that is not a restriction we have to plan around.
+
+Since the content must be a simple string we should use a simple json representation for the quota that contains a list of resources. Representing the quota:
+```yaml
+yunikorn.apache.org/namespace.quota: "{\"cpu\": \"100m\",\"memory\": \"1G\",\"nvidia.com/gpu\": \"1\"}"
+```
+
+Similar as for other resources we allow in annotations: we allow any string as the key content.
+The value content should be interpreted as a Kubernetes formatted resource quantity. Parsing will handle that enforcement. If any of the values do not comply with the formatting no quota will be set.
+Propagation to the core
+No changes are proposed or required. The quota is passed from the k8shim into the core via the application tags. The content of the tag is a Resource object as defined in the scheduler interface. The schedule interface Resource object supports arbitrary resources already. The content passed from the k8shim to the core will not change. There will also be no changes in the way the quota will be processed in the core as that processing is not linked to resource types.
+Backwards compatibility
+The current annotations will remain supported for the 1.x minor releases. Deprecation will be announced with the first release that supports the new annotation. Messages mentioning the processing of the old annotation will also be logged at a WARN level in the logs.
+
+Removing the existing annotation processing is a breaking change that could cause a large change in behaviour. Removal of processing for the old annotations should be part of the next major release. The next major release is 2.0.0. This is based on the fact that we do not have a deprecation policy defined as yet.
+
+Preference in processing will be with the new annotations. In the case that both the old and new annotations are present on the namespace the new annotation will be used. Using both old and new annotations, i.e. merging of the two sets, will not be supported.
diff --git a/versioned_docs/version-1.5.0/design/historical_usage_tracking.md b/versioned_docs/version-1.5.0/design/historical_usage_tracking.md
new file mode 100644
index 0000000..83d0680
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/historical_usage_tracking.md
@@ -0,0 +1,762 @@
+---
+id: historical_usage_tracking
+title: Historical Usage Tracking
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+## Introduction
+
+Tracking an application lifecycle is not easy in the current YuniKorn setup.
+Logs and statedump data is required to build an overview.
+Historical tracking of applications is only possible with limited detail per application.
+Applications come and go, we only track what is running and a time limited set of completed applications.
+
+Storing detailed tracking information in memory is problematic.
+Unbound growth can cause performance issues or even an out of memory failure of the service.
+The lessons learned from the YARN days have shown that it is not simple.
+
+A conscious choice was also made to keep YuniKorn stateless.
+The service does not maintain its own datastore for any of its operations.
+Adding a datastore for application and usage tracking will complicate things.
+
+Tracking data can be easily generated and made available for external consumption.
+The design will discuss the option to generate an event stream for applications, queues and nodes to allow external tools to build an overview of the usage pattern.
+
+Work to be tracked under [YUNIKORN-1628](https://issues.apache.org/jira/browse/YUNIKORN-1628)
+
+## Goals
+
+- Implementation of an event stream for an application, including:
+ - State changes
+ - Asks and allocation changes
+- Implementation of an event stream for a node, including:
+ - State changes
+ - Allocation changes
+- Implementation of an event stream for a queue, including:
+ - State changes
+ - Usage changes
+- Define a REST interface for event retrieval
+
+## Non Goals
+
+- Add a data store for the historical data
+- Display the event information
+- Rebuild data on recovery
+ - Historical data will not be rebuild
+- Authentication and Authorisation on the REST interface
+
+## Existing Event System
+
+The event system was designed to be far more flexible than the current usage.
+Events for requests, applications, nodes and queues have been defined.
+Most of those are currently not used.
+The current event system is built on top of a generic event definition.
+The scheduler interface defines the event as:
+
+```go
+si.EventRecord {
+ Type: recordType,
+ ObjectID: String,
+ GroupID: String,
+ Reason: String,
+ Message: String,
+ TimestampNano: int64,
+}
+```
+
+Events definitions are used in the core and shim.
+The simplicity of the events will most likely not match the current requirements.
+
+As part of the standard scheduling cycle we track details at different levels.
+Prometheus' metrics are tracked for specific changes and updates.
+We also generate events, REQUEST events, that get passed back into the K8shim to attach to the Kubernetes pods.
+
+The current events that get passed back are for:
+
+- *Insufficient Queue Resources*: part of the allocation cycle. Called from: `Application.tryAllocate()`
+- *Placeholder Size Mismatch*: part of the placeholder replacement cycle. Called from: `Application.tryPlaceholderAllocate()`
+
+Both events are of the type REQUEST.
+The K8shim also implements processing of NODE events but there is no code that generates those types of events.
+The APPLICATION and QUEUE type events are not created or processed.
+
+The event system was added as part of [YUNIKORN-42](https://issues.apache.org/jira/browse/YUNIKORN-42).
+The jira contains a simple design document for the event system.
+It only was implemented for events that we could not at that point get in any other way: [design v2](https://docs.google.com/document/d/1aKfY6wnBPCyBl03UfmMHbTSpabAbfxtsHT2KzOgEOQs/edit).
+The original thought was a more generic event system.
+It would relate everything back to either a pod or the YuniKorn CRD and was focussed on diagnostics and troubleshooting in general: [design v1](https://docs.google.com/document/d/19iMkLJGVwTSq9OfV9p75wOobJXAxR_CRo4YzSRG_Pzw/edit#heading=h.worp3vfnqgtn).
+Linking it all back to a specific pod is The YuniKorn CRD is not used at this point in time.
+
+### Scheduler integration
+
+When the event system was designed the whole purpose was to allow out of band processing of the events from the scheduling cycle.
+The events are generated during the scheduling cycle and processed asynchronously in the event system.
+
+A channel is used for collecting the events during the scheduling cycle.
+The scheduler generates an event and adds it for processing to the channel.
+After the placement of the event on the channel the scheduler proceeds with the normal cycle.
+Processing of the events does not, and must not, block the scheduling cycle.
+
+This part of the event system must be maintained as it will guarantee the performance of the scheduler.
+
+### Event storage
+
+All events that have been sent to the channel are read from the channel and placed in a temporary store for publishing.
+The store is a simple map with the key of the *ObjectID*.
+
+Some of the assumptions in the event store however make it not directly usable for the purpose as described here.
+The main limitation is that there can only be one event in the channel per *ObjectID*.
+The newest one is kept, the older one is dropped when a new event comes in.
+This however does not mean the code already available with minor changes could be re-used for this purpose.
+
+### Publishing events
+
+The current event system uses a push system for event publishing.
+The event system allows the creation of multiple event publishers.
+There is no implementation or design for a pull system, like for instance a REST based interface.
+
+Currently, there is only an event publisher for a shim defined.
+This push system will send all the events that have been collected during the push interval to the shim.
+All events that are pushed will be removed from the store.
+This keeps the size of the store to a minimum.
+
+There is no filtering or post-processing of events implemented.
+Each event stored in the store is forwarded to the shim when the publisher runs.
+See limitations of the store mentioned above.
+In the earlier design a level filter was described.
+That level was never part of the follow-up design, not in the events and not in the processing.
+
+## Event System Updates
+
+The currently defined events are not a fit for the system we want and need.
+Since the events are only used between the core and the K8shim via the publisher we do not need to maintain backwards compatibility.
+Changes can be made to the messages as we do not currently expose the message to the outside.
+
+The *message* and *reason* fields are currently not properly used.
+The content of the two overlaps.
+The message in both cases contains the reason, in slightly different wording.
+We do not need both.
+
+### Event description
+
+We are also missing two fields to allow an extended usage for historical tracking: the *resource* and the *change type*.
+Based on that the new message that would allow using the event system for tracking historical change would be:
+
+```go
+si.EventRecord {
+ Type: eventRecordType,
+ ChangeType: eventChangeType,
+ ChangeDetail: eventChangeDetail,
+ TimestampNano: int64,
+ ObjectID: String,
+ ReferenceID: String,
+ Resource: si.Resource,
+ Message: String,
+}
+```
+
+The existing *GroupID* has been renamed to *ReferenceID*. The
+*ReferenceID* is the identification of the second object for the event.
+As an example that would be the Allocation UUID for a new allocation
+added to an application, request or node. For the queue that would be
+the application ID.
+
+Note that the order of the attributes in the final definition might be
+different as we do not reuse names and IDs in the messages.
+
+### Event types
+
+By keeping the event type for REQUESTS we can still fulfil the original
+design of YUNIKORN-42. The current enumeration for the *eventRecordType*
+would not change. Definition of the eventRecordType enumeration:
+
+```text
+UNKNOWN = 0
+REQUEST = 1
+APP = 2
+NODE = 3
+QUEUE = 4
+```
+
+### Change types
+
+The additional change type that is added allows us to track the type of
+change. Depending on the content of the other fields it provides the
+possibility to track all changes we need to track. Definition of the
+eventChangeType enumeration:
+
+```text
+NONE = 0
+SET = 1
+ADD = 2
+REMOVE = 3
+```
+
+### Change details
+
+Change detail provides more on the reason for the event. The change
+detail is an enumerated value linked to the event types.
+
+```text
+DETAILS_NONE = 0
+
+REQUEST_CANCEL = 100 // Request cancelled by the RM
+REQUEST_ALLOC = 101 // Request allocated
+REQUEST_TIMEOUT = 102 // Request cancelled due to timeout
+
+APP_ALLOC = 200 // Allocation changed
+APP_REQUEST = 201 // Request changed
+APP_REJECT = 202 // Application rejected on create
+APP_NEW = 203 // Application added with state new
+APP_ACCEPTED = 204 // State change to accepted
+APP_STARTING = 205 // State change to starting
+APP_RUNNING = 206 // State change to running
+APP_COMPLETING = 207 // State change to completing
+APP_COMPLETED = 208 // State change to completed
+APP_FAILING = 209 // State change to failing
+APP_FAILED = 210 // State change to failed
+APP_RESUMING = 211; // State change to resuming
+APP_EXPIRED = 212; // State change to expired
+
+NODE_DECOMISSION = 300 // Node removal
+NODE_READY = 301 // Node ready state change
+NODE_SCHEDULABLE = 302 // Node schedulable state change (cordon)
+NODE_ALLOC = 303 // Allocation changed
+NODE_CAPACITY = 304 // Capacity changed
+NODE_OCCUPIED = 305 // Occupied resource changed
+NODE_RESERVATION = 306; // Reservation/unreservation occurred
+
+QUEUE_CONFIG = 400 // Managed queue update or removal
+QUEUE_DYNAMIC = 401 // Dynamic queue update or removal
+QUEUE_TYPE = 402 // Queue type change
+QUEUE_MAX = 403 // Max resource changed
+QUEUE_GUARANTEED = 404 // Guaranteed resource changed
+QUEUE_APP = 405 // Application changed
+QUEUE_ALLOC = 406 // Allocation changed
+ALLOC_CANCEL = 500 // Allocation cancelled by the RM
+ALLOC_PREEMPT = 501 // Allocation preempted by the core
+ALLOC_TIMEOUT = 502 // Allocation cancelled due to timeout
+ALLOC_REPLACED = 503 // Allocation replacement (placeholder)
+ALLOC_NODEREMOVED = 504 // Allocation cancelled, node removal
+```
+
+### Content definition
+
+The *eventRecordType* defines the object that the *ObjectID* points to.
+The content of the *ReferenceID* depends on two things:
+- Change type
+- Object type inferred by the *eventRecordType*
+
+For an object of type application it would not make sense to have the reference point to another application etc.
+At this point the following mapping for *ReferenceID* is assumed:
+
+| Event Type | Reference ID |
+|------------|----------------------------------|
+| REQUEST | ApplicationID |
+| APP | Allocation UUID or Request UUID |
+| NODE | Allocation UUID |
+| QUEUE | ApplicationID or Allocation UUID |
+
+If the *eventChangeType* Is NONE or SET the *ReferenceID* is always empty.
+The exception is for the REQUEST as in that case the change type NONE has an application ID set.
+This special case is used to implement the existing functionality of the event system: sending events to the shim.
+
+The APP record type supports Request IDs and Allocation UUIDs to be set in the *ReferenceID*.
+The type of ID that is referenced is defined by the *ChangeDetail*.
+
+The QUEUE record type supports application IDs and Allocation UUIDs to be set in the *ReferenceID*.
+Individual allocations are not tracked on the queue.
+However, we leave that option open for the event system at this point.
+
+For the QUEUE if *ReferenceID* and *Resource* are set, the ID points to an allocation.
+This can only happen if the *ChangeDetail* is set to QUEUE_ALLOC.
+
+If only the *ReferenceID* is set it points to an application.
+This can only happen if the *ChangeDetail* is set to QUEUE_APP.
+Both cases can use the same *eventChangeType*, add or remove an allocation or an application from a queue.
+
+The *Resource* that is part of any event would be the size of the resource for that change.
+The interpretation depends on the *eventChangeType*.
+For a SET it is the absolute value. For the ADD and REMOVE it is the positive delta on the existing tracked value.
+The *Resource* is always a positive value.
+
+## Event storage
+
+The storage of events needs to be updated to allow multiple events to be stored for an *ObjectID*.
+This is required as one object could trigger multiple events in a short time and all are important to track.
+This will require a change from a map based storage to a different store or a change in the key used for the map.
+
+The simplest solution might be a slice of pointers to events.
+The slice has a configurable, limited set of entries.
+The slice is accessed and maintained as a ring buffer.
+This will prevent an unlimited growth of the schedulers memory requirements.
+During the implementation a decision will be made which configuration, time; count or both, for limiting the size will be supported.
+
+Nil, or empty, events will not be added to the storage.
+However, no publisher must assume that while processing events retrieved from the buffer and needs to handle nil references.
+A location in the buffer could contain a nil pointer and must be skipped while iterating over the buffer.
+If a publisher gets a nil event pointer it must not crash and proceed as normal.
+
+The current design will use two stores: one for external retrieval and one for the shim events.
+A filter at the receiving end will be created.
+The shim events will then be processed as per the existing implementation.
+The new events will be stored and managed as per this design document in [SHIM events](#shim-events).
+
+### Publishing events
+
+Events are currently not post processed and all events are sent to the shim.
+With the newly added events the publishing needs to be split into two separate loops.
+
+#### SHIM events
+
+Current functionality will be supported as described above.
+Updates are required as the content of the events has changed.
+The same event type will be generated.
+The same processing mechanism will be used.
+
+As part of processing events REQUEST type events are special.
+The REQUEST type with the change type NONE has an application ID set and is only sent as an event to the shim.
+These events must not be made available by any other publishers and are stored in a separate ring buffer.
+After the event is sent to the shim the reference to the event in the buffer is cleared, replaced with a nil.
+
+#### REST
+
+The REST service that is currently running can be reused for exposing the events.
+Proposal is to add a new endpoint to the HTTP service.
+The REST endpoint servicing the event data should be a new end point.
+
+Based on the current REST api definition the data exposed in the following new endpoint will be added to expose the events:
+```text
+/ws/v1/events/batch
+```
+
+The design will not define a sophisticated interface for querying events, like filtering, grouping, etc.
+This is a deliberate decision.
+Such an API can encourage bad practices later on.
+That should be done in a separate application which fetches the events from YuniKorn and persists them in a permanent storage and offers a more feature rich REST/query interface.
+This would be similar to what already exists in Apache Hadoop like Job History Server or Application Timeline Service.
+
+As mentioned earlier in this document, such an application is not in the scope.
+
+The batch endpoint, by default, returns a limited number of events.
+The number of events to return can be specifically set via the query parameter *count*.
+If the requested *count* is larger than the available number of events all events are returned.
+
+```text
+/ws/v1/events/batch?count=100
+```
+
+To prevent a DOS attack the maximum value of *count* might be limited as part of the implementation.
+
+The second query parameter that will be supported is *start*.
+This specifies the start ID of the first event returned.
+Every event is assigned a unique id starting from 0.
+The ring buffer maintains the current available highest and lowest ID.
+If *start* refers to an ID which doesn't exist in the buffer, then an empty response is returned from the server, with *LowestID* and *HighestID* properly filled:
+```json lines
+{
+ "InstanceUUID": "b4751b7d-a1a3-4431-bb68-bff896adb9c2",
+ "LowestID": 1100,
+ "HighestID": 11000,
+ "EventRecords": null
+}
+```
+
+In the next request, start should be defined as 1100.
+
+In the response, the *InstanceUUID* field shows the unique instance ID.
+Since YuniKorn is stateless, the generated ID for an event is not saved anywhere.
+If an event consumer saves the events to a backend database, it makes it possible to distinguish between events with the same ID.
+Also, clients can easily detect that YuniKorn was restarted.
+
+#### Streaming
+
+Streaming allows the user to see a continuous stream of events as they are occurring.
+This can be very useful if we want to trace the state changes inside YuniKorn for a longer amount of time.
+The REST interface can return the last "n" amount of events that occurred in a certain timeframe, but it will always be limited.
+We can only increase the number of events at the expense of memory usage, which might not be acceptable in certain environments.
+
+Although streaming is a nice feature, we do not consider it as a "must have", at least not in the first version of the history tracking.
+This is because it's more complicated in nature: it maintains an open HTTP connection towards the client which makes it stateful.
+There are ordering and memory usage concerns&considerations.
+We need to make sure that rogue clients cannot abuse it.
+Therefore, the first release of history tracking focuses on the batch interface.
+
+Streaming needs to coexist beside the current REST api.
+A separate endpoint must be exposed for the event stream:
+```text
+/ws/v1/events/stream
+```
+
+Streaming can be similar to how it's implemented inside Kubernetes informers.
+The API server keeps the connection alive and sends the necessary events about pods, nodes, configmaps, etc.
+The incoming data stream is decoded by the listeners.
+Event processing on the client side is not part of this design.
+
+At this point we do not provide endpoints for consumers to stream a specific event type as defined in [Event types](#event-types).
+This could be made available via separate endpoints in the future following the same design as for *batch* specification
+
+The *stream* endpoint does not take any query parameters.
+The consumer must allow for more than one (1) event to be sent in one response.
+No information besides the events will be sent in the response.
+
+As an example below the approximate output for the stream endpoint for two events
+
+```json
+[
+ {
+ "type": 2,
+ "changeType": 1,
+ "changeDetail": 203,
+ "timestamp": 1649167576110750000,
+ "objectID": "spark-app-1"
+ },
+ {
+ "type": 4,
+ "changeType": 2,
+ "changeDetail": 405,
+ "timestamp": 1649167576110754000,
+ "objectID": "root.spark",
+ "referenceID": "spark-app-1"
+ }
+]
+```
+
+## Event overview
+
+### Generic events
+
+The following events are generated for streaming and REST consumers, meaning that they will not be sent to the shim.
+This is based on the description of the [Event System Updates](#event-system-updates) earlier in this document.
+It serves as a reference for the core scheduler actions that will trigger the event.
+
+| Event type | Change type | Change details | Reference type | Notes |
+|------------|-------------|-------------------|----------------|--------------------------------------------------|
+| APP | ADD | DETAILS_NONE | | New application added |
+| APP | ADD | APP_ALLOC | AllocationID | Successful allocation |
+| APP | ADD | APP_REQUEST | RequestID | Incoming resource request (pod) |
+| APP | REMOVE | DETAILS_NONE | | Normal removal of application |
+| APP | REMOVE | APP_REJECT | | Application rejected |
+| APP | REMOVE | ALLOC_CANCEL | AllocationID | Normal removal requested by the shim |
+| APP | REMOVE | ALLOC_TIMEOUT | AllocationID | Timeout |
+| APP | REMOVE | ALLOC_REPLACED | AllocationID | Replacement (placeholder) |
+| APP | REMOVE | ALLOC_PREEMPT | AllocationID | Preemption triggered |
+| APP | REMOVE | ALLOC_NODEREMOVED | AllocationID | Removal triggered by node removal |
+| APP | REMOVE | APP_REQUEST | RequestID | Normal removal requested by the shim |
+| APP | REMOVE | REQUEST_TIMEOUT | RequestID | Timeout (placeholder) |
+| APP | REMOVE | REQUEST_CANCEL | RequestID | Removal triggered by application removal |
+| APP | SET | APP_NEW | | State change: New |
+| APP | SET | APP_ACCEPTED | | State change: Accepted |
+| APP | SET | APP_STARTING | | State change: Starting |
+| APP | SET | APP_RUNNING | | State change: Running |
+| APP | SET | APP_COMPLETING | | State change: Completing |
+| APP | SET | APP_COMPLETED | | State change: Completed |
+| APP | SET | APP_FAILING | | State change: Failing |
+| APP | SET | APP_FAILED | | State change: Failed |
+| APP | SET | APP_RESUMING | | State change: Resuming |
+| APP | SET | APP_EXPIRED | | State change: Expired |
+| | | | | |
+| NODE | ADD | DETAILS_NONE | | New node added to the cluster |
+| NODE | ADD | NODE_ALLOC | AllocationID | Successful allocation |
+| NODE | REMOVE | NODE_DECOMISSION | | Removal requested by the shim |
+| NODE | REMOVE | NODE_ALLOC | AllocationID | Normal allocation removal requested by the shim |
+| NODE | SET | NODE_READY | | Update "ready" status |
+| NODE | SET | NODE_SCHEDULABLE | | Update "schedulable" status |
+| NODE | SET | NODE_CAPACITY | | Update node capacity |
+| NODE | SET | NODE_OCCUPIED | | Update occupied resources |
+| NODE | ADD | NODE_RESERVATION | | Add reservation to a node |
+| NODE | REMOVE | NODE_RESERVATION | | Remove reservation from a node |
+| | | | | |
+| QUEUE | ADD | DETAILS_NONE | | Adding new configured queue |
+| QUEUE | ADD | QUEUE_DYNAMIC | | Adding new dynamic queue |
+| QUEUE | ADD | QUEUE_APP | ApplicationID | Application added |
+| QUEUE | REMOVE | DETAILS_NONE | | Removing configured queue |
+| QUEUE | REMOVE | QUEUE_DYNAMIC | | Removing dynamic queue |
+| QUEUE | REMOVE | QUEUE_APP | ApplicationID | Application removed |
+| QUEUE | SET | QUEUE_MAX | | Max resource changed |
+| QUEUE | SET | QUEUE_GUARANTEED | | Guaranteed resource changed |
+| QUEUE | SET | QUEUE_TYPE | | Queue type change(parent/leaf) |
+
+### Shim events
+
+Shim events are a type of REQUEST and the application ID is set.
+Right now we send only two types of events to the shim, both of which are originated from the application:
+
+- Insufficient resources in the queue for a request
+- Real allocation is larger than placeholder
+
+| Event type | Change type | Change details | Reference type | Notes |
+|------------|-------------|----------------|----------------|-----------------------|
+| REQUEST | NONE | DETAILS_NONE | ApplicationID | New application added |
+| REQUEST | NONE | DETAILS_NONE | ApplicationID | Successful allocation |
+
+## Configuration
+
+As part of the design we anticipate the following configuration to be added for the event storage and retrieval.
+All settings will have the shared prefix: *service.event*.
+All the mentioned settings will require a restart of the scheduler for updates to take effect.
+
+The following settings will be added to the system:
+
+A flag to enable or disable sending request events to the K8shim.
+This externalises the current setting that needs a recompile of the code.
+The default mirrors the current default in the code.
+
+- Name: **requestEventsEnabled**
+- Allowed values: all boolean values will be converted into a boolean using ParseBool as defined in the `strconv` package[^1].
+- Default value: false
+
+A flag to enable or disable event collection in the system.
+The default is enabled for the tracking events.
+
+- Name: **trackingEventsEnabled**
+- Allowed values: all boolean values will be converted into a boolean using ParseBool as defined in the `strconv` package.
+- Default value: true
+
+Size of the store for events that will be sent to the shim, request events.
+Note that setting a size of 0 effectively turns off the request event system.
+
+- Name: **requestStoreCapacity**
+- Allowed values: integer value that can be converted into a 32-bit integer using ParseUint as defined in the `strconv` package[^2].
+- Default value: 1,000
+
+The number of events to store in the ring buffer.
+This number defines the size of the ring buffer and thus the memory impact the event storage will have on the scheduler.
+Note that setting a size of 0 effectively turns off the event collection system.
+
+- Name: **ringBufferCapacity**
+- Allowed values: integer value that can be converted into a 32-bit integer using ParseUint as defined in the `strconv` package
+- Default value: 100,000
+
+The maximum number of events to return in one REST response.
+The maximum that could be returned is limited to the ring buffer capacity.
+However, preparing a large response the size of the whole ring buffer could cause large peaks in the schedulers memory usage.
+This setting should be used to prevent these peaks.
+
+- Name: **RESTResponseSize**
+- Allowed values: integer value that can be converted into a 32-bit integer using ParseUint as defined in the `strconv` package
+- Default value: 10,000
+
+## Performance considerations
+
+### Memory usage of historical elements
+
+We need to estimate how many elements we can expect to be generated with different pod churn rates on a daily basis.
+Queue and node events are much rarer, so they are not accounted for.
+We also ignore potential failures and scheduling errors.
+
+An application undergoes state transitions, so the following events are generated:
+- Add new application
+- State change: New
+- State change: Accepted
+- State change: Starting
+- State change: Running
+- State change: Completing
+- State change: Completed
+- Allocation changed
+
+Pod (request) events:
+- Incoming allocation request
+- Successful allocation on a node
+
+Pod (request) events with gang scheduling:
+- Incoming resource request
+- Successful allocation on a node
+- Replace event for placeholder
+
+| Number of pods | Number of apps | Pods per app | Job type | Events per app | Events per pod | ∑ apps | ∑ pods | ∑ events |
+|----------------|----------------|--------------|----------|----------------|----------------|-------------------|-----------------|----------|
+| 150k | 1500 | 100 | normal | 200 + 7 | 2 | 310k (1500 * 207) | 300k (2 * 150k) | 610k |
+| 150k | 1500 | 100 | gang | 300 + 7 | 3 | 460k (1500 * 307) | 450k (3 * 150k) | 910k |
+| 300k | 3000 | 100 | normal | 200 + 7 | 2 | 621k (3000 * 207) | 600k (2 * 300k) | 1,2M |
+| 300k | 3000 | 100 | gang | 300 + 7 | 3 | 921k (3000 * 307) | 900k (3 * 300k) | 1,8M |
+
+On a busy cluster with 150,000 - 300,000 pods per day, we can expect the number of events generated to be around 600k - 1,8M (depending on the scheduling style).
+
+If we want to retain the events for 5 days, we need to store 9 million events in the worst case and 3 million in the best case.
+
+With 9 million objects in the memory, it is also critical to estimate how much extra memory usage is expected from YuniKorn with different kinds of data structures:
+
+- slice of pointers to si.EventRecord objects `[]*si.EventRecord`
+- slice of si.EventRecord objects `[]si.EventRecord`
+- slice of a memory-optimised data type `[]internalRecord`
+
+As the name suggests, internalRecord type would only be available inside the ring buffer.
+It's used together with string interning to save memory caused by repeated strings like IDs (application ID, ask ID, allocation ID, etc.).
+There are various interning libraries in Go, but removing elements is not possible from them.
+We need to create our own, so we can track the number of references per string.
+
+The following table summarizes memory usage and GC time, depending on how we store the events.
+The test was executed multiple times to get averages and ranges.
+The slice was created once, then an index variable was increased for each record, so the built-in `append()` function was not used.
+
+The results clearly indicate two things:
+
+- having millions of pointers has negative performance on both the memory usage and GC load
+- batch workloads inherently have a lot of repeated strings, so interning makes a noticeable difference
+
+The GC detects pointers, since it has to follow them to walk the object graph to identify unreferenced objects.
+If elements in the slice are pointers, then those allocations are scattered all around the heap (bad locality) with header information added by the allocator.
+This reduces memory access times and increases the amount of the data that is allocated.
+
+In the test, the assumption was 100 pods per application.
+In real life, this value might be different, most likely it's lower.
+The lower the value, the smaller the advantage of the interning - on the extreme, every application consists of only one pod.
+But even with only 30 pods per application, it has value - more modest, 20-30% reductions are not negligible when storing millions of elements.
+
+| Object type | Number of elements | Increased memory usage\* | GC time\*\* |
+|-------------------|--------------------|--------------------------|-------------|
+| []*si.EventRecord | 3M | 805 MiB | 85-105 ms |
+| []si.EventRecord | 3M | 437 MiB | 41-55 ms |
+| []internalRecord | 3M | 211 MiB | 5-16 ms |
+| []*si.EventRecord | 6M | 1595 MiB | 160-200 ms |
+| []si.EventRecord | 6M | 856 MiB | 75-110 ms |
+| []internalRecord | 6M | 404 MiB | 10-30 ms |
+| []*si.EventRecord | 9M | 2380 MiB | 270-320 ms |
+| []si.EventRecord | 9M | 1280 MiB | 116-150 ms |
+| []internalRecord | 9M | 593 MiB | 16-33 ms |
+
+*\* "Sys" metrics returned by the Go runtime*
+
+*\*\* measured by the execution time of `runtime.GC()`*
+
+### Handling slow and rogue clients during streaming
+
+It's possible to overwhelm YuniKorn with streaming or execute a DoS attack.
+
+The first thing we need to address is a rogue client which simply stops reading on the receiver side from the TCP socket.
+Eventually writes will block in YuniKorn.
+If we use channels and per-client goroutines, this should not be an issue - we can always check if the buffer is full using the built-in len() function.
+If this happens, we can just drop the connection.
+In some situations, the buffer is expected to be full, for example, when sending history.
+It's always faster to send out a lot of existing objects, so we can expect the buffer to get full at times.
+This can be solved by repeatedly checking the rate of writes - if it falls below a certain threshold, then again, we just drop the connection with an error message.
+
+We can also limit the number of concurrent streaming connections and historical objects to send, e.g. we simply won't serve requests which would result in sending millions of elements.
+
+Note that it's still possible to deny the service from legitimate users because YuniKorn lacks authentication on the REST interface.
+This will be solved in the future.
+
+### Approaches to streaming
+
+When a client requests a stream of events, it can define how many past events it wants to receive.
+This is necessary because when YuniKorn starts up, it's not possible to connect immediately.
+It can be important for certain users to save the state of YuniKorn on an external storage completely from the start.
+
+Sending the historical events must be separated from the new events that are constantly occurring in the system, that is, old and current events must not interleave.
+For example: we want a stream of old and new events, this means:
+
+- We're currently at *event~n~*
+- Send out events from *event~0~* all the way up to *event~n~*
+- In the meantime, *k* number of events have been generated
+- Therefore, send events up to event*~n+k~* until we're caught up completely
+
+#### **Send history first, then stream new events**
+
+The first approach is to retrieve the objects from the ring buffer and start to send them on a channel to the receiver.
+There is a separate buffered channel for new events with capacity *localBufferSize*.
+As soon as we finish sending the history, we switch to this channel and relay events from it towards the client.
+
+We can express this with the following pseudocode:
+```go
+consumerEventLoop(elementCount) {
+ local = initLocalEventChannel(localBufferSize)
+ // send history
+ history := ringbuffer.getHistory(elementCount)
+ foreach h : history {
+ receiver <- h
+ }
+ // bridge new events
+ foreach e : <-local {
+ receiver <- e
+ }
+}
+```
+
+The publisher logic is:
+```go
+publishEvent(event) {
+ foreach c : localChannels {
+ if len(c) == localBufferSize { // local buffer full?
+ closeChan(receiver)
+ closeChan(local)
+ receiverId = getReceiver(c)
+ abort("slow receiver: " + receiverId)
+ }
+ c <- event
+ }
+}
+```
+
+Each receiver has its own goroutine.
+That is, 10 clients means 10 different `consumerEventLoop()` calls running on separate goroutines.
+
+To detect slow clients, we simply check the value of `len(c)` whenever we submit the new event.
+If this equals the capacity of the buffer, we drop the receiver and close the connection.
+
+It is possible to send events directly to the receiver channel and skip `c` on the publisher side which is local inside consumerEventLoop.
+This requires a simple state machine and a temporary slice of new events, since we are not allowed to send directly as long as the history is being transmitted.
+Overall, this is more complicated than the one described above.
+
+The simplicity of this method is notable, which is the main advantage: reasoning about its correctness is trivial.
+We do not need complicated test code and back-and-forth review cycles to ensure that it indeed results in the correct order of events.
+
+The biggest question is what happens if channel local becomes full.
+Let's say that localBufferSize is 1000.
+On a busy cluster with 100 events / second (busiest period), it takes 10 seconds to fill it up completely.
+However, nothing stops us from setting the capacity of the buffer to a higher value.
+If we bump it up to 10000, the client is expected to read the history in 100 seconds.
+On a decent network, this amount of time should be enough to read the event history, even if it means 100 000 entries from the ring buffer.
+If a single si.EventRecord results in 300 bytes of JSON text, then it is around 30MiB of data.
+Even if we take serialisation into account, this will not (can not) take minutes.
+
+The biggest disadvantage of this method is when a user requests a lot of elements from the history.
+For example, someone wants to retrieve the last 1M elements.
+In this case, we need to allocate a slice for 1M entries.
+Based on the measurements, this will generate a sudden spike in the live heap size, around 145-150 MiB. If people repeatedly do this, the memory allocation of YuniKorn can increase very quickly, possibly resulting in a termination of the YuniKorn pod by the kubelet.
+
+#### **Stream new events directly from ring buffer**
+
+Instead of notifying receivers directly, we just insert the data into the ring buffer.
+For every client, we create a cursor which points to a given position inside the buffer.
+If the pointer is behind the tail, it just keeps reading and going forward until it reaches the current tail.
+
+Although this method sounds simple on the surface, the implementation is complicated:
+
+- When sending the history, we can't rely on `len(buf)` checking.
+It is because when we are sending the history, we are always very close to the capacity.
+Unlike the previous solution, we are processing elements from the buffer directly, and we do not bridge two channels.
+If we do that, we just add further complications.
+Therefore, we have to calculate the sending rate, at least while sending past objects.
+If the pointer is not making progress (or just very slow), then the tail of the ring buffer will catch up to it, but it can take a very long time.
+- In case of multiple cursors, we have to maintain which one is the closest to the tail.
+If we don't do that, we have to check every cursor when adding elements.
+We cannot wait for a slow reader to jump to the next element, so we have to invalidate the cursor, then have to calculate which is the nearest to tail again.
+- Whenever we reach the last element, we have to block and utilize the well-known wait-notify pattern (`Cond.Broadcast()` and `Cond.Wait()` in Go).
+- The overall sending logic is much more complicated: keep reading batches from the ring buffer until we reach the last element, then switch to wait-notify and block.
+
+The advantage of this method is that we don't need large channel buffers to make sure that they don't get filled while sending historical records.
+If the reader is fast enough, we just keep marshalling `si.EventRecord` objects and bump the pointer to the next element.
+
+[^1]: https://pkg.go.dev/strconv#ParseBool
+
+[^2]: https://pkg.go.dev/strconv#ParseUint
diff --git a/versioned_docs/version-1.5.0/design/interface_message_simplification.md b/versioned_docs/version-1.5.0/design/interface_message_simplification.md
new file mode 100644
index 0000000..b6766ce
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/interface_message_simplification.md
@@ -0,0 +1,309 @@
+---
+id: interface_message_simplification
+title: Simplifying Interface Messages
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+# Simplifying Interface Messages and Breaking Shim build dependency on Core
+
+# Proposal
+This document describes a) complexity hidden behind existing Interface messages and
+explains the newly defined SI messages and its dependent changes on Core and Shim.
+b). Breaking Shim build dependency on Core.
+
+## Goals
+The goal is to provide the same functionality before and after the change.
+- unit tests before and after the merge must all pass.
+- Smoke tests defined in the core should all pass without major changes definition.
+- End-to-end tests that are part of the shim code must all pass without changes.
+## Background
+The current interface allows us to only send one message between a shim and the core. This provides us with a really simple way of interactions definition.
+
+The complexity is however hidden in the message itself. Every message serves multiple purposes and when the message is received the core and shim need to unpack it and process each part separately and for certain parts in a real specific order.
+Because the message serves a number of purposes it has a large overhead. This might not show up in the code directly as the heavy lifting is done in the generated code. It will show up in the amount of data as a message, even if it does not have all fields, still needs to be encoded in a way that it unpacks correctly on the other side.
+
+## Simplifying Interface Messages
+
+Proposal is to split the one large message into 3 separate messages - one for each entity:
+
+- Allocations
+- Applications
+- Nodes
+
+### API Interface Changes
+
+```
+package api
+
+import "github.com/apache/incubator-yunikorn-scheduler-interface/lib/go/si"
+
+type SchedulerAPI interface {
+ // Register a new RM, if it is a reconnect from previous RM, cleanup
+ // all in-memory data and resync with RM.
+ RegisterResourceManager(request *si.RegisterResourceManagerRequest, callback ResourceManagerCallback) (*si.RegisterResourceManagerResponse, error)
+
+ // Update Allocation status
+ UpdateAllocation(request *si.AllocationRequest) error
+
+ // Update Application status
+ UpdateApplication(request *si.ApplicationRequest) error
+
+ // Update Node status
+ UpdateNode(request *si.NodeRequest) error
+
+ // Notify scheduler to reload configuration and hot-refresh in-memory state based on configuration changes
+ UpdateConfiguration(clusterID string) error
+}
+
+// RM side needs to implement this API
+type ResourceManagerCallback interface {
+
+ //Receive Allocation Update Response
+ UpdateAllocation(response *si.AllocationResponse) error
+
+ //Receive Application Update Response
+ UpdateApplication(response *si.ApplicationResponse) error
+
+ //Receive Node update Response
+ UpdateNode(response *si.NodeResponse) error
+
+ // Run a certain set of predicate functions to determine if a proposed allocation
+ // can be allocated onto a node.
+ Predicates(args *si.PredicatesArgs) error
+
+ // RM side implements this API when it can provide plugin for reconciling
+ // Re-sync scheduler cache can sync some in-cache (yunikorn-core side) state changes
+ // to scheduler cache (shim-side), such as assumed allocations.
+ ReSyncSchedulerCache(args *si.ReSyncSchedulerCacheArgs) error
+
+ // This plugin is responsible for transmitting events to the shim side.
+ // Events can be further exposed from the shim.
+ SendEvent(events []*si.EventRecord)
+
+ // Scheduler core can update container scheduling state to the RM,
+ // the shim side can determine what to do incorporate with the scheduling state
+ // update container scheduling state to the shim side
+ // this might be called even the container scheduling state is unchanged
+ // the shim side cannot assume to only receive updates on state changes
+ // the shim side implementation must be thread safe
+ UpdateContainerSchedulingState(request *si.UpdateContainerSchedulingStateRequest)
+
+ // Update configuration
+ UpdateConfiguration(args *si.UpdateConfigurationRequest) *si.UpdateConfigurationResponse
+}
+```
+
+### Interface Changes to replace UpdateRequest
+
+UpdateRequest would be divided into below messages:
+
+#### AllocationRequest
+```
+message AllocationRequest {
+ repeated AllocationAsk asks = 1;
+ AllocationReleasesRequest releases = 2;
+ string rmID = 3;
+}
+```
+#### ApplicationRequest
+```
+message ApplicationRequest {
+ repeated AddApplicationRequest new = 1;
+ repeated RemoveApplicationRequest remove = 2;
+ string rmID = 3;
+}
+```
+#### NodeRequest
+```
+message NodeRequest {
+ repeated NodeInfo nodes = 1;
+ string rmID = 2;
+}
+```
+### Merging Create and Update NodeInfo into Single NodeInfo
+```
+message NodeInfo {
+ enum ActionFromRM {
+ CREATE = 0;
+ UPDATE = 1;
+ DRAIN = 2;
+ SCHEDULABLE = 3;
+ DECOMISSION = 4;
+ }
+
+ string nodeID = 1;
+ ActionFromRM action = 2;
+ map<string, string> attributes = 3;
+ Resource schedulableResource = 4;
+ Resource occupiedResource = 5;
+ repeated Allocation existingAllocations = 6;
+}
+```
+
+### Event Changes to replace UpdateRequest
+
+RMUpdateRequestEvent would be replaced by following events:
+
+- RMUpdateAllocationEvent
+- RMUpdateApplicationEvent
+- RMUpdateNodeEvent
+
+### Interface Changes to replace UpdateResponse
+
+UpdateResponse would be divided into below messages:
+
+#### AllocationResponse
+```
+message AllocationResponse {
+ repeated Allocation new = 1;
+ repeated AllocationRelease released = 2;
+ repeated AllocationAskRelease releasedAsks =3;
+ repeated RejectedAllocationAsk rejected = 4;
+}
+```
+#### ApplicationResponse
+```
+message ApplicationResponse {
+ repeated RejectedApplication rejected = 1;
+ repeated AcceptedApplication accepted = 2;
+ repeated UpdatedApplication updated = 3;
+}
+```
+#### NodeResponse
+```
+message NodeResponse {
+ repeated RejectedNode rejected = 1;
+ repeated AcceptedNode accepted = 2;
+}
+```
+
+### Event Changes for UpdateResponse
+
+Scheduler/Context.go from Core already triggers an event for each entity separately and rmproxy.go is the one which handles all these events and packs it under single *si.UpdateResponse and eventually sends to shim through scheduler_callback#RecvUpdateResponse. With the above API interface change, rmproxy.go would use appropriate callback method to send response to shim. With this separate callback approach, each entity response would be handled separately in shim.
+
+## Detailed Flow Analysis
+
+### Add/Delete Allocations
+
+The RM (shim) sends a simplified AllocationRequest as described above. This message is wrapped by the RM proxy and forwarded to the cache for processing. The RM can request an allocation to be added or removed.
+
+```
+1. Shim sends a simplified AllocationRequest to core through SchedulerAPI.UpdateAllocation
+2. RMProxy sends rmevent.RMUpdateAllocationEvent to scheduler
+3. On receiving the above event, scheduler calls context.handleRMUpdateAllocationEvent to do the
+ following:
+ 3.1: processAsks
+ 2.1.2: Process each request.Asks ask of AllocationRequest request and adds to the application
+ 2.1.2: In case of rejection, triggers RMRejectedAllocationAskEvent with
+ all asks which has been rejected
+ 2.1.2: On receiving RMRejectedAllocationAskEvent, RMProxy.processUpdatePartitionConfigsEvent
+ process the event, creates a AllocationResponse using RMRejectedAllocationAskEvent
+ attributes and send to shim through UpdateAllocation callback method
+ 3.2: processAskReleases
+ 2.2.1: Process each request.Releases.AllocationAsksToRelease ask release of AllocationRequest request
+ and removes from the application
+ 3.3: processAllocationReleases
+ 3.3.1: Process each request.Releases.AllocationRelease allocation release of AllocationRequest
+ request and removes from the application
+ 3.3.2: Collect all above exact released allocations and triggers RMReleaseAllocationEvent with all allocations needs to be released
+ 3.3.3: On receiving RMReleaseAllocationEvent, RMProxy.processRMReleaseAllocationEvent
+ prcoess the event, creates a AllocationResponse using RMReleaseAllocationEvent
+ attributes and send to shim through UpdateAllocation callback method
+ 3.3.4: Collect all above confirmed (placeholder swap & preemption) allocations
+ and send to shim through two ways
+ a). Wraps confirmed allocations as AssumedAllocation
+ and send to shim ReSyncSchedulerCache callback plugin
+ b). Wraps confirmed allocations as Allocation and triggers
+ RMNewAllocationsEvent. On receiving RMNewAllocationsEvent,
+ RMProxy.processAllocationUpdateEvent process the event, creates a
+ AllocationResponse using RMNewAllocationsEvent attributes and send to shim
+ through UpdateAllocation callback method
+```
+
+### Add/Delete Applications
+
+The RM (shim) sends a simplified ApplicationRequest as described above. This message is wrapped by the RM proxy and forwarded to the cache for processing. The RM can request an application to be added or removed.
+
+```
+1. Shim sends a simplified ApplicationRequest to core through SchedulerAPI.UpdateApplication
+2. RMProxy sends rmevent.RMUpdateApplicationEvent to scheduler
+3. On receiving the above event, scheduler calls context.handleRMUpdateApplicationEvent to do the
+ following:
+ 3.1: Add new apps to the partition.
+ 3.1.2: Wraps AcceptedApps and RejectedApps (if any) as part of RMApplicationUpdateEvent
+ and fires the same
+ 3.1.2: On receiving RMApplicationUpdateEvent, RMProxy.processApplicationUpdateEvent
+ process the event, creates a ApplicationResponse using RMApplicationUpdateEvent
+ attributes and send to shim through UpdateApplication callback method
+ 3.2 Remove apps from the partition.
+ 3.2.1: Collect all allocations belongs to the removed app and triggers
+ RMReleaseAllocationEvent with all allocations needs to be released
+ 3.2.2: On receiving RMReleaseAllocationEvent, RMProxy.processRMReleaseAllocationEvent
+ prcoess the event, creates a AllocationResponse using RMReleaseAllocationEvent
+ attributes and send to shim through UpdateAllocation callback method
+```
+
+### Add/Delete Nodes
+
+The RM (shim) sends a simplified NodeRequest as described above. This message is wrapped by the RM proxy and forwarded to the cache for processing. The RM can request an node to be added or removed.
+
+```
+1. Shim sends a simplified NodeRequest to core through SchedulerAPI.UpdateNode
+2. RMProxy sends rmevent.RMUpdateNodeEvent to scheduler
+3. On receiving the above event, scheduler calls context.handleRMUpdateNodeEvent to do the
+ following:
+ 3.1: Add new node to the partition.
+ 3.1.2: Wraps AcceptedNodes and RejectedNodes (if any) as part of RMNodeUpdateEvent
+ and fires the same
+ 3.1.2: On receiving RMNodeUpdateEvent, RMProxy.processRMNodeUpdateEvent
+ process the event, creates a NodeResponse using RMNodeUpdateEvent
+ attributes and send to shim through UpdateNode callback method
+ 3.2: Update node
+ 3.2.1 Update the partition resource
+ 3.3: Drain node
+ 3.3.1 Ensures node is not schedulable
+ 3.4: Decommissioning (Remove) node from the partition.
+ 3.4.1: Ensures node is not schedulable
+ 3.4.2: Collect all above exact released allocations from that node and triggers
+ RMReleaseAllocationEvent with all allocations needs to be released
+ 3.4.3: On receiving RMReleaseAllocationEvent,
+ RMProxy.processRMReleaseAllocationEvent process the event, creates a
+ AllocationResponse using RMReleaseAllocationEvent attributes and
+ send to shim through UpdateAllocation callback method
+ 3.4.4: Collect all above confirmed (placeholder swap & preemption) from that node
+ allocations and send to shim through two ways
+ a). Wraps confirmed allocations as AssumedAllocation and send to shim
+ through ReSyncSchedulerCache callback plugin
+ b). Wraps confirmed allocations as Allocation and triggers RMNewAllocationsEvent.
+ On receiving RMNewAllocationsEvent, RMProxy.processAllocationUpdateEvent
+ process the event, creates a AllocationResponse using RMNewAllocationsEvent
+ attributes and send to shim through UpdateAllocation callback method
+```
+
+## Breaking the Shim build dependency on Core
+
+Planned for different phases.
+
+### Phase 1
+Moved all plugins from core to appropriate place in SI under ResourceManagerCallback,
+a single common interface.
+
+### Phase 2
+Please refer https://issues.apache.org/jira/browse/YUNIKORN-930 for more details
diff --git a/versioned_docs/version-1.5.0/design/preemption.md b/versioned_docs/version-1.5.0/design/preemption.md
new file mode 100644
index 0000000..0106bdd
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/preemption.md
@@ -0,0 +1,581 @@
+---
+id: preemption
+title: Preemption
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Preemption Design
+
+## Introduction
+
+This design is created to replace the preemption code that has been part of
+YuniKorn for a long time.
+
+The original code was never fully tested. After the core cache removal as part
+of [YUNIKORN-317](https://issues.apache.org/jira/browse/YUNIKORN-317) the
+preemption code was taken out of the scheduling path. The only changes that
+have gone into the preemption code were compilation fixes. As part of the
+preparation for the new preemption design the old code was removed in
+[YUNIKORN-1269](https://issues.apache.org/jira/browse/YUNIKORN-1269).
+
+Preemption of workloads is a feature used by almost all schedulers to help
+critical workloads run. The criticality can be based on the fact that the
+system cannot work without it, i.e. DaemonSets on K8s, or based on some
+other factors like SLA or priority. Preemption for DaemonSet pods was
+implemented as part of
+[YUNIKORN-1085](https://issues.apache.org/jira/browse/YUNIKORN-1085). This
+design thus focuses on preemption for all other workloads.
+
+## Goals
+
+* Re-use existing Kubernetes objects
+* Implement inter-queue preemption
+* Priority knowledge inside the core scheduler to support preemption
+
+## Non Goals
+
+* Intra-queue preemption
+* Cross node preemption
+* Full priority support
+
+## Preemption and Priorities
+
+### Kubernetes
+
+All pods for the entire cluster, across namespaces, are sorted based on
+priority for scheduling. A higher-priority will get scheduled before a
+lower-priority pod. Preemption within the Kubernetes scheduler is also
+based solely on the priority of the pod that is being scheduled. The
+full documentation can be found
+[here](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/).
+
+The simple rule that governs both scheduling order and preemption behavior is
+that a pod with a higher priority is more important than a pod with a lower
+priority. This crosses namespaces. There are no boundaries or limitations
+that can be set. Pods from a namespace with a higher priority are scheduled
+earlier and could trigger the preemption of a pod from another namespace.
+
+Priority of a pod is defined in the pod specification. Priority is a number,
+but is referenced in the pod specification via a PriorityClass object. The
+object defines a named priority and maps it to a value. In more recent
+versions (Kubernetes 1.24 and later), the PriorityClass can also contain a
+preemption policy for the scheduling phase. This preemptionPolicy only allows
+the pod to opt-out from preempting other lower priority pods during the
+scheduling phase. It cannot, and is not designed to, alter preemption behavior
+while the pod is running.
+
+The same pod priority is also used during the pod eviction by the kubelet on
+the node. For example, if the node has been overloaded and runs out of memory
+or CPU, the node can evict pods. These evictions help to keep the node stable.
+Pods considered for eviction are ranked on priority among some other factors.
+
+### YuniKorn
+
+The behavior in Kubernetes is all focused on the scheduling cycle. When
+looking at preemption from the standpoint of a batch or data processing
+workload, we need to take into account the possibility to opt-out while
+running. This option does not exist in the current configuration for workloads.
+
+The simple approach for preemption fits in with the service type workloads
+that Kubernetes is designed for. Services handle the scale up and or scale
+down that could be triggered by preempting one of the pods for that service.
+
+This is not the case when we look at a Spark job as an example. The driver pod
+is the manager of the other pods that belong to the same job. When the driver
+gets preempted the whole job is impacted. Preempting a driver has a follow-on
+effect that could cause multiple pods to be removed from the cluster. For a
+large job that could mean hundreds of pods. The impact is far larger than a
+single pod. Preempting a manager or originator pod should be prevented as much
+as possible.
+
+The other case which we need to account for is an interactive session that
+runs on the cluster. This interactive session, like a python notebook, is a
+one-off instance and restarts have a wider impact. Preempting an instance
+like a notebook is not a desirable outcome either.
+
+Building in all the smarts to know which pods to avoid is difficult, or even
+impossible. For all these cases the option to allow a “do not preempt me” flag
+on the pod would be required. The design will discuss this option in the Pod
+specification.
+
+### Priority limitations
+
+PriorityClass objects which define the priority in a Kubernetes cluster are
+cluster-wide objects. The cluster administrator defines the objects and thus
+the supported priorities.
+
+There are no limitations on the priority that can be set on a pod. When a pod
+is submitted the only check performed is that the PriorityClass specified as
+part of the pod specification is defined. Any rogue user can create a pod with
+the highest defined priority.
+
+Limiting priorities is possible but requires an API server flag which is
+difficult to support in public cloud environments. We cannot rely on the option
+to be available in a cluster scheduled by YuniKorn.
+
+## The Laws of Preemption
+
+Prior knowledge of implementing preemption in the Apache YARN scheduler has
+been used to create this design. It forms the basis for the "laws of preemption".
+These laws, or rules, describe overall preemption behavior.
+
+Preemption for YuniKorn is based on the hierarchical queue model and guaranteed
+resources assigned to a queue.
+
+Preemption is used to get the usage of a queue to at least its guaranteed
+resource capacity. It cannot guarantee that every queue will get its
+guaranteed resource capacity. There are cases that make it impossible to get to
+that state. Preemption therefore does not guarantee a final optimal state can
+or will be reached. The end state of the queues will normally thus be a usage
+that is approximately the guaranteed resource capacity.
+
+An example case that makes it impossible to reach the optimal state is when the
+sum of the guaranteed resources for a group of queues is larger than the current
+resources available in the cluster. It is impossible to get all queues their
+guaranteed resources. More cases like this exist in a hierarchy with maximum
+quotas set at a certain point in the hierarchy preventing the optimal state from
+being reached.
+
+First an overview of the laws, then the explanation and rationale behind each
+of the rules:
+
+1. Preemption policies are strong suggestions, not guarantees
+2. Preemption can never leave a queue lower than its guaranteed capacity
+3. A task cannot preempt other tasks in the same application
+4. A task cannot trigger preemption unless its queue is under its guaranteed
+ capacity
+5. A task cannot be preempted unless its queue is over its guaranteed capacity
+6. A task can only preempt a task with lower or equal priority
+7. A task cannot preempt tasks outside its preemption fence (one-way constraint)
+
+Most of the rules given around tasks and queues are there to prevent a
+preemption storm or loop. Preempting a task that could trigger a follow-up
+preemption should be avoided. The rule described should warrant against that.
+Breaking the rules could lead to unpredictable behavior either during
+scheduling or application runs.
+
+### Preemption policies are strong suggestions, not guarantees
+
+A pod is able to request not to be preempted. However, this is not a guarantee.
+There could be circumstances that the scheduler might still preempt the pod.
+This will be a last resort action if no other solution is available. This means
+that the preemption policy is a strong suggestion, not a guarantee.
+
+The example use case that could break the rule and preempt a pod that opted out
+of preemption is the DaemonSet case. The DaemonSet pod must run on the specified
+node. If all pods that are running on that node are either other DaemonSet pods
+or pods that have opted out of preemption there is no other choice. A pod that
+opted out of preemption will be preempted.
+
+### Preemption can never leave a queue lower than its guaranteed capacity
+
+Guaranteed resource capacities configured on a queue are the optimal resources
+for the queue from a preemption point of view. Since preemption attempts to get
+a queue at least its guaranteed resources, allowing preemption to reduce a queue
+below its guaranteed resource capacity would trigger a further round of
+preemption.
+
+When a task is considered for preemption the removal of the task from the
+queue's usage must not leave the queue below its guaranteed resource capacity.
+If the removal of the task would drop the queue below its guarantee that task
+cannot be a candidate.
+
+This could mean that a queue, even when it is above its guarantee, still does
+not allow preempting any of the tasks in that queue.
+
+### A task cannot preempt other tasks in the same application
+
+As part of the current design we are only implementing inter-queue preemption.
+This rule does not apply for inter-queue preemption. It is documented here to
+make extending to intra-queue preemption possible without a rule change.
+
+When a task is considered for preemption the task that triggered the preemption
+cannot be from the same application.
+
+### A task cannot trigger preemption unless its queue is under its guaranteed capacity
+
+Preemption attempts to get a queue at least its guaranteed resources. Allowing
+preemption for a task in a queue that is already over its guaranteed resource
+capacity does not help reaching that goal.
+
+Preemption is not the tool to get a queue to its full maximum resource
+capacity. For that reason no tasks in a queue are allowed to trigger
+preemption in this case. Normal scheduling will handle the assignment of
+resources for queues above their guaranteed capacity.
+
+### A task cannot be preempted unless its queue is over its guaranteed capacity
+
+This is an extension to the rule that prevents a queue from going below its
+guaranteed resource capacity. If a queue is already below its guaranteed
+capacity, no tasks running in that queue can be considered for preemption.
+The queue can already trigger preemption of tasks from other queues. Allowing
+preemption to remove more resources from the queue would cause the queue to be
+even further from its ideal usage.
+
+### A task can only preempt a task with lower or equal priority
+
+This rule is linked to the default preemption behavior of Kubernetes. There is
+however a slight difference when compared to the Kubernetes rule. Kubernetes
+only allows pods to preempt other pods with a lower priority, not with equal
+priority.
+
+Preemption in YuniKorn has a slightly broader application as we also use it to
+allow a queue to use its guaranteed resource capacity. The equality has been
+added to allow pods from the same priority running in two different queues to
+preempt each other. Without that equality allowance on priorities that
+redistribution would be slow and difficult.
+
+### A task cannot preempt tasks outside its preemption fence
+
+This last rule is a specific multi-tenancy rule. YuniKorn allows setting up a
+preemption fence on a queue that will limit the possible tasks that can be
+considered for preemption. A task can only preempt another task if it runs on
+a queue that is located inside the preemption fence. More on preemption
+fencing below.
+
+## Preemption Fence
+
+In a cluster that runs workloads for multiple tenants preemption should not be
+allowed to preempt a workload from one tenant and give the resources to
+another. A tenant in this case does not have to be a company, it could be a
+division or business group within the same company. Tenants map to the queue
+hierarchy. The queue hierarchy can thus cross tenant boundaries.
+
+To translate this to a YuniKorn centric view: confining preemption to assess
+only a part of the queue hierarchy should be possible. For this we want to
+introduce a preemption fence. The preemption fence is a setting on the queue
+which stops preemption from looking at queues outside the fence boundary.
+
+The preemption fence can be set on any queue at any point in the hierarchy.
+It blocks traversing up the queue hierarchy when looking for a victim to
+preempt. The fence is a unidirectional fence. It prevents going out (i.e.
+higher up the queue hierarchy), but does not prevent coming in (or down) the
+queue hierarchy. The below diagram shows an example hierarchy with preemption
+fencing applied:
+
+
+
+The example has three preemption fences defined at different levels. The
+arrows describe the preemption flows.
+
+First look at `queue A`.
+A fence is defined on the leaf queue which means that preemption is fenced to
+that queue. This would limit tasks to find a preemption victim within the
+queue. This in effect turns off inter-queue preemption for the leaf
+`queue A`. This would be a case for intra-queue preemption only. Note that in
+the current design intra-queue preemption is excluded, and we will not
+implement this use case. However, we do allow the configuration to be set.
+
+The second fence is set up at the `tenant 1` level. This fence has an impact
+on the task running in `queue B`. The tasks in `queue B` can find a victim
+to preempt in all queues below the `tenant 1` queue (shown in the diagram with
+the _yellow_ arrows).
+
+A similar setup is created for the `tenant 2` queue. However, in this case
+there can be preemption from `queue 1` to `queue 2` and vice-versa (shown in
+the diagram with the _blue_ arrows).
+
+None of the tasks running in the queues below `tenant 1` or `tenant 2` can
+preempt a task from the `system` queue. Tasks in the `system` queue are not
+fenced. That means that a task in the `system` queue can check any queue in
+the hierarchy (shown in the diagram as the _red_ arrows).
+
+## Preemption Logic
+
+For preemption to be triggered by the default Kubernetes scheduler, the pod
+must fit within any namespace quota and no node can be found to allocate the
+pod on. Preemption tries to find a node to run the pod on and is focused on
+nodes only. All pods on a node with a lower priority are considered candidates
+for preemption. The scheduler reduces the candidates to a victim list just
+large enough to fit the new pod.
+
+The scheduler tracks the node undergoing preemption in the pod that triggered
+the preemption. It does not guarantee, for a number of reasons, that the pod
+that triggered the preemption also gets scheduled on the node the pods were
+preempted from.
+
+From the YuniKorn side we already have a node focused preemption for DaemonSet
+pods. A pod from a DaemonSet must run on a specific node. Preemption is
+focused on making space on that node. YuniKorn guarantees that the pod will
+run on that node and nothing else will be scheduled on that node until the
+DaemonSet pod is scheduled.
+
+Generic preemption in YuniKorn is linked to a queue that is under its
+guaranteed quota (**rule 4**). The preemption is only triggered after a delay.
+A request must have been pending for at least the preemption delay before
+preemption will be considered for it. Instead of looking at a node the first
+step is to find victims based on the queue that is over its guaranteed quota
+(**rule 5**). If no queues can be found within the preemption fence
+(**rule 7**), preemption immediately stops.
+
+The next step is to build a list of victims. This could be from one or
+multiple queues. Any pod with a priority higher (**rule 6**) than the pod
+triggering the preemption is discarded. The next step is to discard any pod
+with resources larger than the over guarantee portion (**rule 2**). Proximity
+of the victim queue in the hierarchy to the originating queue that triggers
+the preemption might be considered when we decide which victims to preempt.
+Details will be worked out during the implementation phase.
+
+Cross node preemption is not supported. Any set of victims that get preempted
+must run on the same node. The victims are split into groups per node. A group
+of victims, combined with the allocatable space on the node, will decide if
+the group is a viable group. Checks must be run to make sure that the pod
+will fit the node before starting the preemption. If more than one group is
+viable a choice will need to be made. The logic for choosing a group is an
+implementation detail.
+
+After choosing a group and indirectly a node, the node is reserved for the
+pod. Preemption for the victim list will be triggered and on completion the
+pod will be scheduled on the node. The reservation guarantees that the pod
+triggering the preemption will be scheduled on that node. As a side effect
+the preempted pods that restart can also not “slip back” into the freed up
+space on the node while the scheduler waits for all preempted pods to exit.
+They could be scheduled on a different node but not stop the pod that
+triggered the preemption from being scheduled.
+
+## Preemption Loop Prevention
+
+The rules defined prevent looping and circular preemptions. Testing should
+be added to cover each rule and combinations like below.
+
+ReplicaSets are a good example to look at for looping and circular
+preemption. Each time a pod from a replica set is removed the ReplicaSet
+controller will create a new pod to make sure the set is complete. That
+auto-recreate could trigger loops if the rules are not covering all cases
+correctly. The case described below should be covered as part of standard
+testing.
+
+Example setup: Replica set _Prod Repl_ runs in queue _Prod_. For testing
+a replica set _Test Repl_ runs in the queue _Test_. Both queues belong
+to the same parent queue (they are siblings). The pods all run with the
+same settings for priority and preemption. There is no space left on the
+cluster.
+
+Example 1: _Prod_ is under its guaranteed quota and multiple pods of the
+replica set are pending. All pods for the _Test Repl_ set are running and
+_Test_ is above its guaranteed resource capacity.
+
+Preemption flow case 1: To make room for a _Prod Repl_ pod, a pod from
+the _Test Repl_ set is preempted. Both queues will end up above their
+guaranteed resource. The _Test Repl_ pod is recreated. It cannot preempt
+the _Prod Repl_ pod as that would leave the _Prod_ queue below its
+guaranteed resource capacity. The _Test Repl_ pod will have to wait until
+resources become available in the cluster. The other pods for the
+_Prod Repl_ set will also have to wait. The _Prod_ queue is now above its
+guaranteed resource and cannot trigger preemption anymore.
+
+Preemption flow case 2: _Test_, although above its guaranteed resource
+capacity, would end up below its guarantee if a pod from _Test Repl_ would be
+preempted. No preemption happens and the _Prod Repl_ pods will stay pending
+until space becomes available in the cluster.
+
+Preemption flow case 3: To make room for the _Prod Repl_ pod a pod from the
+_Test Repl_ set is preempted. The _Prod_ queue, even with the additional pod,
+is still below its guaranteed resource. The _Test Repl_ pod is recreated.
+_Prod_ is not considered as a queue with victims as it is still below its
+guaranteed resources. The _Test Repl_ pod will have to wait until resources
+become available in the cluster. The _Prod_ queue can trigger further
+preemptions. Preemption checks start from scratch. Depending on the state this
+could trigger none or more preemptions from the Test queue.
+
+## Preemption Configuration
+
+### PriorityClass
+
+As part of the pod specification we need to be able to provide the ability to
+opt-out of preemption. Allowing a user to specify the opt-out directly on the
+pod independent of the priority causes a disconnect between Kubernetes and
+YuniKorn preemption configuration. It also removes control from the
+administrator of the cluster to define which priorities should not be able
+to opt out.
+
+One of the goals of the design is to not introduce new objects in Kubernetes.
+Combined with the above control and management of the opt-out functionality
+at a cluster level the design reuses the existing PriorityClass object.
+We cannot and must not break existing functionality of the default Kubernetes
+scheduler when we do this. The plugin version of Kubernetes relies on the
+default scheduler and its behavior. Pods will use the priorityClassName just
+like standard Kubernetes.
+
+The PriorityClass object allows specifying a _preemptionPolicy_ with two
+possible values:
+* PreemptLowerPriority
+* Never
+
+The values are hard-coded in the core types and validations. Adding a new
+value is not possible without making a Kubernetes change. That means we cannot
+reuse the _preemptionPolicy_ field itself. The content of the field will be
+propagated to the scheduling core to allow YuniKorn to take the setting into
+account while scheduling.
+
+Just like all other objects in Kubernetes, the PriorityClass object extends
+the base object and includes metadata. As part of the metadata we can set
+labels and or annotations on the object. Labels are more restrictive than
+annotations. As a generic rule we prefer to use annotations over labels for
+all the extensions we are adding.
+
+The proposal is to add an annotation to the PriorityClass that allows an
+opt-out of preemption. The name of the annotation would follow the same
+structure as we use for all other annotations:
+ `yunikorn.apache.org/allow-preemption`
+
+As the value of the annotation the proposal is to use a boolean value:
+* "true": allow the pod to be preempted,
+* "false": do not allow preemption
+If no value is set for _allow-preemption_ the default value _true_ is used.
+
+The design comes with a caveat. The value of "_false_" for the annotation
+on the PriorityClass provides the possibility for a pod to request not to
+be preempted. However, this is not a guarantee. There could be
+circumstances that the scheduler might still preempt the pod. This will be
+a last resort action if no other solution is available. This means that the
+preemption policy as defined above is a strong suggestion, not a guarantee.
+
+The example use case that could break the rule and preempt a pod with
+_allow-preemption_ set to _false_ is the DaemonSet case. The DaemonSet pod
+must run on the specified node. If all pods that are running on that node
+are either other DaemonSet pods or pods that have _allow-preemption_ set
+to _false_ there is no other choice. A pod that opted out of preemption
+will be preempted.
+
+### Application Settings
+
+Priority is part of the pod specification on Kubernetes. Every pod is
+converted into an AllocationAsk by the k8shim. As part of the DaemonSet
+preemption that was implemented in
+[YUNIKORN-1085](https://issues.apache.org/jira/browse/YUNIKORN-1085) the
+priority of a pod is read and set on the AllocationAsk that is created. The
+priority that is passed on currently is just a plain integer value, we do not
+need more than that.
+
+As part of preemption we need two additional values to be passed on. These two
+values together form the preemptionPolicy for YuniKorn. The values must be
+communicated as part of the AllocationAsk sent from the Kubernetes shim to the
+core. Both values are mapped to the content of the PriorityClass described above.
+Instead of adding two separate fields, wrapping them into a new message will
+provide more clarity.
+
+The preemption policy can be processed for any AllocationAsk, even if it has
+the `Originator` flag in the message set to _false_. Specifying different
+preemption behavior for different asks within an application could be important
+to allow low-cost asks to be preempted while high-cost asks opt-out.
+
+The use case is again a Spark application. You want the driver pod to opt-out
+from preemption and, possibly, preempt other pods during scheduling. An executor
+should do neither.
+
+In protobuf message form that would look like:
+
+```
+message AllocationAsk {
+ ...
+ // The preemption policy for this ask
+ PreemptionPolicy preemptionPolicy = 12;
+}
+
+message PreemptionPolicy {
+ // Allow preemption while running
+ // Based on the YuniKorn annotation from the PriorityClass object
+ bool allowPreemption = 1;
+ // Preemption behavior while scheduling
+ // Based on the K8s preemptionPolicy from the PriorityClass object
+ string schedulerPreemption = 2;
+}
+```
+
+### Queue Configuration
+
+Two new attributes will be added to the queue configuration object to allow
+specifying the preemption fence type and preemption delay. Both properties,
+`preemption.policy` and `preemption.delay`, can be set on any queue. The
+attributes are part of the queue properties as follows:
+
+```yaml
+queues:
+ - name: fencedqueue
+ properties:
+ preemption.policy: fence
+ preemption.delay: 30s
+```
+
+Properties are fully supported in child templates. Support for setting these
+properties for dynamic queues is included as part of the existing
+functionality.
+
+The _preemption.policy_ property can be set on the root queue but has no effect.
+The root queue as the top of the hierarchy has no parent. This means preemption
+cannot traverse further up the tree. No messages will be logged if the
+_preemption.policy_ is set on the root queue.
+
+The supported values for the _preemption.policy_ are:
+* default
+* fence
+* disabled
+
+If the value `fence` is set on a queue, tasks running in or below the queue on
+which the property is set cannot preempt tasks outside the queue tree. The
+value `default` does not limit preemption to the subtree. If no policy is set
+the value `default` is used.
+
+A _preemption.delay_ can only be set on a leaf queue. The leaf queue contains
+the tasks. Tasks trigger the preemption in combination with a queue being below
+its guaranteed resource capacity. Setting a preemption delay on a queue that is
+not a leaf has no effect. No messages will be logged if a _preemption.delay_ is
+set on a parent queue.
+
+The _preemption.delay_ value must be a valid Golang duration in string form. The
+value will be converted into a duration using `ParseDuration` as defined in the
+time package.
+
+A _preemption.delay_, if specified, must be larger than `0s`. If no
+_preemption.delay_ property is specified in the queue the default of `30s` is
+used. If parsing of the string fails the default of `30s` is used.
+
+## Scheduler storage object changes
+
+### AllocationAsk
+
+In line with the changes for the communication the objects in the scheduler also
+need to be modified to persist some of the detail communicated. For the
+AllocationAsk we need to store both new fields.
+
+Proposed new fields: _allowPreemption_ and _schedulerPreemption_
+
+ - _schedulerPreemption_ is only used during the scheduling phase.
+ - _allowPreemption_ will be propagated to the resulting allocation as defined
+ below.
+
+### Allocation
+
+The allocation that gets created based on the AllocationAsk needs a new field.
+This removes the need to check the underlying AllocationAsk if it allows
+preemption or not. The scheduler preemption value is not required in the
+Allocation.
+
+Proposed new field: _allowPreemption_
+
+### Queue
+
+The new preemption policy and delay values will need to be stored in the Queue
+object. The configuration object does not change as we use the properties of
+the Queue in the configuration to store them.
+
+Proposed new fields: _preemptionPolicy_ and _preemptionDelay_
diff --git a/versioned_docs/version-1.5.0/design/priority_scheduling.md b/versioned_docs/version-1.5.0/design/priority_scheduling.md
new file mode 100644
index 0000000..84f9ad7
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/priority_scheduling.md
@@ -0,0 +1,408 @@
+---
+id: priority_scheduling
+title: Priority Scheduling
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Currently, YuniKorn can sort at two levels: the applications in a queue and
+the resource requests within the application. The queue sorting policy is
+configurable and supports a _Fair_ or _FIFO_ policy. _StateAware_ is
+considered a filtered _FIFO_ policy. Applications cannot be sorted based
+on priority.
+
+Within an application, requests are sorted based on priority of the requests.
+For requests that have the same priority the submission time is used as a
+secondary sort key. Requests are sorted in _FIFO_ order if they have the
+same priority. Note that the priority of requests was not propagated from
+the shim to the core until
+[YUNIKORN-1235](https://issues.apache.org/jira/browse/YUNIKORN-1235) was
+implemented in v1.0.0. Until that implementation all requests were created
+with the _default_ priority.
+
+Additionally, Kubernetes uses priority as a sort when performing preemption.
+YuniKorn currently only does preemption of requests if a node-specific task
+needs to be scheduled (i.e. a DaemonSet pod,
+[YUNIKORN-1085](https://issues.apache.org/jira/browse/YUNIKORN-1085)).
+
+## Goals
+
+- Attempt to be compatible with Kubernetes standard priority handling
+ wherever possible
+- Priority should be orthogonal to application sorting policies (i.e.
+ _Fair_, _FIFO_ and _StateAware_ policies should support prioritization)
+- It should be possible to limit the scope of priority handling
+- Design of priority scheduling should consider the impact on preemption
+ design
+
+## Non Goals
+
+- Design of preemption
+- Change AllocationAsk sorting within an application
+
+## Definitions
+
+The following terms are defined as follows and will be used throughout this
+document.
+
+### Priority
+
+Priority is a numeric value associated with an Ask. Higher-priority asks have
+higher numeric values. To be compatible with Kubernetes, all 32-bit signed
+integers are considered valid priorities. An ask without a specific defined
+priority is assumed to have the default priority.
+
+### Minimum Priority
+
+Minimum priority is defined as the lowest possible priority, or
+`-2,147,483,648` (- 2<sup>31</sup>).
+
+### Maximum Priority
+
+Maximum priority is defined as the highest possible priority, or
+`2,147,483,647` (2<sup>31</sup> - 1). Note that Kubernetes does not permit
+custom priority classes (see below) to define priorities greater than
+`1,000,000,000` (1 billion). Values higher are reserved for system-level
+priorities.
+
+### Default Priority
+
+Default priority is defined as the value `0`.
+
+### PriorityClass
+
+A [PriorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass)
+is a Kubernetes object which maps between a priority name and an integer
+priority value. The name cannot start with `system-` as that prefix is
+reserved for Kubernetes internal usage.
+
+Additionally, a `preemptionPolicy` may be set on a PriorityClass. Official
+GA support was added in Kubernetes 1.24. The preemption policy is only
+used while scheduling the pod itself. It is not taken into account after
+the pod is scheduled.
+
+The policy defaults to `Never`, implying that pods of this priority will
+never preempt other pods. `PreemptLowerPriority` is also possible, which
+allows the scheduler to preempt pods with lower priority values.
+
+When a pod is submitted with its `priorityClassName` set to a valid priority
+class, a Kubernetes-provided admission controller will automatically update
+the `priority` and `preemptionPolicy` fields of the pod to match those of the
+referenced PriorityClass. Pods that specify a priority as part of the
+specification are rejected if the integer value set on the pod does not match
+the PriorityClass value.
+
+#### Pre-defined PriorityClass objects
+- system-cluster-critical: `2,000,000,000`
+- system-node-critical: `2,000,000,100`
+
+## Current state
+
+The framework for priority is minimally implemented within YuniKorn. The
+_AllocationAsk_ structure within the scheduler interface already contains a
+_priority_ field which is populated by the shim based on the _priority_ field
+on a Pod.
+
+One priority can be set on creation of the _AllocationAsk_. Each
+_AllocationAsk_ within an application may have its own priority.
+_AllocationAsk_ sorting is based on this priority. This means that within the
+application asks or pods are sorted based on priority. High-priority asks will
+be scheduled before low-priority asks.
+
+Priorities are currently also used for the minimal DaemonSet preemption process
+implemented in [YUNIKORN-1085](https://issues.apache.org/jira/browse/YUNIKORN-1085).
+Possible victims are sorted based on their priority among other things. The
+priority of the _Allocation_ is the same as the _AllocationAsk_ they
+represent.
+
+One use of the capability to set a priority for each _AllocationAsk_ could be to
+enable dynamic allocations such as those utilized by Apache Spark. An
+application can submit a set of asks at high priority that will be scheduled
+as soon as possible, and a set of lower-priority asks that can be used for
+additional capacity if available.
+
+## Proposal
+
+### Priority fencing
+
+By default, priority applies across queues globally. In other words, a
+higher-priority ask will be attempted to be satisfied before a lower-priority
+ask regardless of which queue the asks reside in.
+
+However, it is often useful to limit the scope of priority handling. For
+example, it may be desirable to limit the ability of asks in one area of the
+queue hierarchy from jumping ahead of asks in another area. To support this,
+we add the concept of a _Priority Fence_. A queue (either _leaf_ or _parent_)
+may be configured as priority-fenced, which prevents priorities from
+propagating upward.
+
+If a queue is fenced, asks within that queue (and subqueues) will be
+prioritized relative to each other, but not relative to asks in other portions
+of the queue hierarchy. A priority fence is a YuniKorn queue subtree boundary
+above which priorities are no longer considered. A priority fenced queue does
+not expose priority levels of its children to its parent queue, thereby
+ensuring that priorities are only considered within that queue and its
+descendants.
+
+Priority fences may be nested; for example a fenced queue `tenant 1` may have
+child queues `queue A` (fenced) and `queue B` (unfenced). In this case tasks
+in `queue A` will show as the default priority and will not be prioritized
+above `queue B`, but tasks in `queue B` may be prioritized over those in
+`queue A`. In no case will tasks in either queue be prioritized over tasks
+outside the scope of queue `tenant 1`, as they show as the default priority
+(shown in the diagram below with yellow arrows).
+
+A similar setup is created for the `tenant 2` queue. However in this case the
+priorities from `queue 1` to `queue 2` and vice-versa are fully visible
+(shown in the diagram below with blue arrows).
+
+None of the tasks running in the queues below `tenant 1` or `tenant 2` can be
+prioritized above the `system` queue. Tasks in the `system` queue are not
+fenced. That means that a task in the `system` queue can be prioritized above
+any queue in the hierarchy (shown in the diagram below with red arrows).
+
+
+
+### Priority offset
+
+Priorities need to be pre-defined at the cluster level via the PriorityClass
+objects. At the Kubernetes level there is no way to limit which priorities can
+be used in which part of the system. The only exception to that rule are the
+pre-defined system priorities from the Kubernetes system itself. This does not
+give an administrator a lot of control.
+
+Prioritizing a specific queue above another queue based on the priorities of
+the tasks inside the queue is thus difficult. Any user can start a task with
+any priority anywhere in the cluster. To allow an administrator more control
+we need to be able to steer, or augment, the priority of a queue. To
+accomplish this, we introduce the `priority.offset` property of a queue.
+
+The `priority.offset` property on a queue allows the administrator to increase
+(or decrease) a queue's priority. By using the offset a queue can be boosted
+to become a high-priority queue. It also can be used to decrease the priority
+compared to its siblings.
+
+As a general rule: the priority of a _leaf_ queue is equal to the maximum
+priority of the AllocationAsks in that queue, plus the `priority.offset`
+value. The priority of a _parent_ queue is the maximum of the priority of its
+child queues plus the `priority.offset` value. This means that priority
+offsets are additive when traversing up the queue hierarchy. A priority offset
+may be any valid signed `int32` value. The offset is applied after the dynamic
+priority for the queue is calculated.
+
+As a note of caution: a side effect of a very large offset value for a queue
+could result in all pods in that queue being clamped at the maximum priority
+level or exceeding the priority of Kubernetes system priority classes. This
+could then adversely impact the scheduling of node and or cluster critical
+pods in sibling queues. This effect needs to be clearly documented. Logging a
+warning about a configured value that high should also be considered.
+
+A second option that was considered is limiting the offset value to
+`999,999,999`. This would prevent the offset priority from exceeding the
+pre-defined system priorities. This would be a safe option but does not take
+into account the fact that we can have negative priorities that we want to
+offset. The current choice is to go with the first option to document and log.
+
+Fencing affects the priority calculation for the queue. When a queue is
+fenced (i.e. the `priority.policy` is set to `fence`), queue priority
+calculations are disabled for that queue. The result of application priority
+or child queue calculations are ignored for a fenced queue. The priority of
+the fenced queue is always set to the default priority or to the priority
+offset, if specified.
+
+### Extended application sorting
+
+Applications are currently sorted based on one of the defined application
+sorting policies. All application sorting policies only take into account
+applications that have pending requests.
+
+Currently defined are _Fair_, _FIFO_, and _StateAware_:
+
+- _Fair_ sorting sorts based on relative usage within a queue
+- _FIFO_ sorts applications by creation time in a first-in, first-out manner
+- _StateAware_ is similar to _FIFO_ except that only a single "new"
+ application is allowed at once (this is an over-simplification, but will
+ suffice here)
+
+To support priority scheduling, the policies _Fair_ and _FIFO_ will get an
+equivalent policy that considers the priority of an application first. For
+applications that have the same priority, sorting will then fall back to the
+secondary sort criteria (_Fair_ or _FIFO_).
+
+_StateAware_ as a special form of _FIFO_ will however always filter
+applications first and then sort. A priority based version of _StateAware_
+must change the way it filters. The single "new" application that is
+considered can no longer be the oldest application. That would be a major
+behavioral change from the current policy.
+
+The existing policies will not be changed to maintain backwards
+compatibility. None of the existing policies have a tiebreak built in. In
+the case the comparison returns equality the order of the elements is not
+changed.
+
+Instead of introducing new policies to add priority to the existing
+behavior a flag will be added that allows switching priorities on or off.
+The property will be added to the queue.
+
+New property: `appication.sort.priority`
+
+### Extended queue sorting
+
+Queue sorting currently only supports one policy: _Fair_. Queues cannot
+be sorted using a _FIFO_ mechanism. _FIFO_ sorting of queues is not in
+scope of this design. However the design has allowed for a simple addition
+of a new policy.
+
+The _Fair_ policy works similarly to application _Fair_ sorting, except
+that it sorts based on usage ratios between two queues. Allocations are
+compared to guaranteed resources. This results in the queue furthest below
+its guaranteed resource to be first in the sorted queue list as it is
+considered "more starved". For queues that have the same usage ratio,
+the queue with the highest pending requests is considered more starved.
+
+To add priority support, a new property will be added to consider the
+current queue priority as the main sorting factor. To break the tie between
+queues with the same priority, starvation like in the _Fair_ policy will be
+used. Again breaking the tie for queues with the same _Fair_ usage using the
+pending requests.
+
+The existing policy will not be changed to maintain backwards compatibility.
+The same property as used for application sorting will be reused to control
+priority behavior.
+
+New property: `application.sort.priority`
+
+### Priority Configuration
+
+#### Queue Configuration
+
+Three new attributes will be added to the queue configuration object to allow
+specifying the priority fence type and priority offset and priority sorting.
+All properties `priority.policy`, `priority.offset`, and
+`application.sort.priority`, can be set on any queue. The attributes are part
+of the queue properties as follows:
+
+```yaml
+queues:
+ - name: fencedqueue
+ properties:
+ priority.policy: fence
+ priority.offset: 100
+ application.sort.priority: enabled
+```
+
+The `priority.policy` and `priority.offset` fields are not inherited from the
+parent queue to a child queue; they apply only to the queue they are specified
+on.
+
+Note that the `application.sort.priority` field is only inherited if the value
+is set to `disabled` (the `enabled` value is already the default). This allows
+priorities to be ignored for an entire queue subtree by setting the property
+on the parent queue.
+
+Properties are fully supported in child templates. Support for setting these
+properties for dynamic queues is included as part of the existing
+functionality.
+
+The supported, not case sensitive, values for the `application.sort.priority`
+are:
+
+ - `enabled`
+ - `disabled`
+
+The proposal is to default to `enabled` for the `application.sort.priority`
+property. Given that the Kubernetes default scheduler and preemption support
+are all based on priorities that choice seems more logical than `disabled`.
+
+The `application.sort.priority` when set on a _parent_ queue affects the way
+the queues are sorted. It will change queue sorting from the standard _Fair_
+sorting to priority-based sorting. To break the tie for equal priority
+queues it will use the _Fair_ comparison. When set on a _leaf_ queue it will
+change the application sorting itself. All three existing policies will be
+supported with `application.sort.priority` set to `enabled` or `disabled`.
+For the `disabled` case nothing will change and the old behavior is
+maintained. For the `enabled` case the primary sort will be priority and the
+configured policy (_Fair_, _FIFO_, or _StateAware_) will be used as
+tiebreakers.
+
+The `priority.policy` and `priority.offset` can be set on the `root` queue
+but have no effect. The root `queue` as the top of the hierarchy has no
+parent. This means priority cannot traverse further up the tree. No messages
+will be logged if the `priority.policy` or `priority.offset` are set on the
+`root` queue.
+
+The supported, not case sensitive, values for the `priority.policy` are:
+
+- `default`
+- `fence`
+
+If the value `fence` is set on a queue, the queue on which the property is
+set does not propagate its priority outside of the queue's subtree. The value
+`default` does not fence the queue. If no policy is set the value `default` is
+used.
+
+The `priority.offset` value must be a valid integer in string form. The value
+will be converted into an integer using the standard parsing methods as defined
+via: [`strconv.ParseInt`](https://pkg.go.dev/strconv#ParseInt)`(s, 10, 32)`.
+
+If no `priority.offset` property is specified in the queue the default of `0`
+is used. Specifying the property with an empty value is equivalent to `0`. If
+parsing of the string fails the default of `0` is used.
+
+### Scheduler storage object changes
+
+#### Application
+
+The _Application_ object within the scheduler core will be updated with a
+dynamic priority value, defined as the maximum priority of all pending asks.
+To avoid sorting performance degradation, this value should be cached and
+recomputed only when allocation asks are added, allocated, or removed.
+
+This recalculation can be avoided in many instances. When new asks are added,
+an application's priority need not be recalculated unless the priority of the
+new ask is greater than that of the application as a whole. Similarly, when
+removing an ask, the priority need not be adjusted unless the ask's priority
+is equal to the application's priority. When recalculating priority due to
+adding a high-priority ask, the application's priority can simply be set to
+the new ask's priority. When recalculating due to removal of an ask of equal
+priority to the application, we can stop considering additional asks as soon
+as any other ask of equal priority is encountered. This means that the
+performance impact of application priority updates will be `O(1)` when adding
+an ask, and at worst `O(n)` when removing an ask.
+
+New field: `askMaxPriority`
+
+#### Queue
+
+The new configuration values need to be maintained on the queue. Besides those
+values the _Queue_ object within the scheduler core will be updated with a
+dynamic priority value. This dynamic priority will be defined as the maximum
+priority of all applications within the queue, plus any "priority offset"
+configured.
+
+To avoid sorting performance degradation, this value should be cached and
+recomputed only when necessary. For a _leaf_ queue, this is required if
+application priorities in the queue have changed, and for a _parent_ queue this
+is required if the priorities of child queues have changed.
+
+New fields: `priorityPolicy`, `priorityOffset`, `currentPriority`
+
diff --git a/versioned_docs/version-1.5.0/design/resilience.md b/versioned_docs/version-1.5.0/design/resilience.md
new file mode 100644
index 0000000..ac0c93b
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/resilience.md
@@ -0,0 +1,144 @@
+---
+id: resilience
+title: Resilience
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This is not a HA (High-availability) design, HA implies that a service can
+survive from a fatal software/hardware failure. That requires one or more
+standby instances providing same services to take over active instance on failures.
+Resilience here means for YuniKorn, we can restart it without losing its state.
+
+## The problem
+
+YuniKorn is designed as a stateless service, it doesn't persist its state, e.g
+applications/queues/allocations etc, to any persistent storage. All states are
+in memory only. This design ensures YuniKorn to be able to response requests with
+low latency, and deployment mode is simple. However, a restart (or recovery) will
+have the problem to lose state data. We need a decent way to reconstruct all
+previous states on a restart.
+
+## Design
+
+### Workflow
+
+Scheduler core has no notion of "state", which means it does not know if it is under recovering.
+It is too complex to maintain a series of `scheduler states` in both core and shim, because we must
+keep them in-sync. However, if we live under a simple assumption: **scheduler core only responses
+requests, the correction of requests is ensured by shim according its current state**.
+The design becomes much simpler. This way, the shim maintains a state machine like below. When
+it is under `running` state, it sends new requests to the scheduler core as long as a new one is found;
+when under `recovering` state, it collects previous allocations and send recovery messages to
+the scheduler core, and waiting for recovery to be accomplished.
+
+Shim scheduler state machine
+
+```
+ Register Recover Success
+New -----------> Registered -----------> Recovering ----------> Running
+ | Fail
+ --------> Failed
+```
+
+Following chart illustrate how yunikorn-core and shim works together on recovery.
+
+
+
+Restart (with recovery) process
+- yunikorn-shim registers itself with yunikorn-core
+- shim enters "recovering" state. Under "recovering" state, the shim only scans existing nodes and allocations, no new scheduling requests will be sent.
+ - shim scans existing nodes from api-server and added them to cache
+ - shim scans existing pods from api-server, filter out the pods that already assigned (scheduled to a node), and added that to cache (allocation in that node)
+ - shim sends update request to yunikorn-core with the info found in previous steps
+- yunikorn-core handles update requests, the steps should look like a replay of allocation process, including
+ - adding node
+ - adding applications
+ - adding allocations
+ - modifying queue resources
+ - update partition info
+- when all nodes are fully recovered, shim transits the state to "running"
+- shim notifies yunikorn-core that recovery is done, then yunikorn-core transits to "running" state.
+
+### How to determine recovery is complete?
+
+Shim queries K8s api-server to get how many nodes were available in this cluster. It tracks the recovering status of each node.
+Once all nodes are recovered, it can claim the recovery is completed. This approach requires us to add `recovering` and `recovered`
+states to nodes' state machine in the shim.
+
+### Node recovery
+
+In the shim layer, it maintains states for each node and pods running on this node. When start to recover nodes,
+all nodes initially are considered as under `recovering`. Only when all pods running on this node are fully recovered,
+the node can be considered as `recovered`.
+
+
+
+Like demonstrated on upon diagram,
+
+- Node0 is still recovering because pod0 is recovering.
+- Node1 is recovered (become schedulable) because all pods on this node have been recovered.
+- Node2 is lost, shim lost contact with this node. If after sometime this node comes back, shim should still try to recover this node.
+
+### Requests for recovery
+
+During recovery process, shim needs to collect all known information of applications, nodes and allocations from the underneath
+Resource Manager and use them for recovery.
+
+#### Applications
+
+Existing applications must be recovered first before allocations. Shim needs to scan all existing applications
+from nodes, and add applications info as a list of `AddApplicationRequest` in the `UpdateRequest`. This is same
+as the fresh application submission.
+
+```
+message AddApplicationRequest {
+ string applicationID = 1;
+ string queueName = 2;
+ string partitionName = 3;
+}
+```
+
+#### Nodes and allocations
+
+Once a shim is registered to the scheduler-core, subsequent requests are sent via `UpdateRequest#NewNodeInfo`
+(see more from [si.proto](https://github.com/apache/yunikorn-scheduler-interface/blob/master/si.proto)).
+The structure of the messages looks like,
+
+```
+message NewNodeInfo {
+ // nodeID
+ string nodeID = 1;
+ // optional node attributes
+ map<string, string> attributes = 2;
+ // total node resource
+ Resource schedulableResource = 3;
+ // existing allocations on this node
+ repeated Allocation existingAllocations = 4;
+}
+```
+Shim needs to scan all existing allocations on a node and wrap these info up as a `NewNodeInfo`, add that to a
+`UpdateRequest` and then send to scheduler-core.
+
+**Note**: the recovery of existing allocations depend on the existence of applications, which means applications must
+be recovered first. Since scheduler-core handles `UpdateRequest` one by one, it is required that all existing allocations
+in a `UpdateRequest` must from known applications or new applications embedded within the same `UpdateRequest`, which can be
+specified in `NewApplications` field. Scheduler-core ensures `NewApplications` are always processed first.
+
diff --git a/versioned_docs/version-1.5.0/design/scheduler_configuration.md b/versioned_docs/version-1.5.0/design/scheduler_configuration.md
new file mode 100644
index 0000000..54b616a
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/scheduler_configuration.md
@@ -0,0 +1,246 @@
+---
+id: scheduler_configuration
+title: Scheduler Configuration
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The Yunikorn core scheduler configuration has two separate areas that need to be configured. The scheduler service itself, things like web service ports etc, and the queue configuration. The split between the two types of configuration is proposed with two points in mind:
+* Separation of duty
+* Dynamic vs Static
+
+The scheduler configuration is mainly static. There is no need to change a web service port or a scheduling policy while the service is running. The queue configuration is far more dynamic and can change while the service is running.
+
+From a separation of duty we can allow an operator that manages the cluster to make changes to the scheduler queues. You would not want to allow that administrator to change the scheduler configuration itself.
+
+Separated from the core scheduler configuration we have one or more shim configurations. We currently cannot anticipate the deployment model of the scheduler and its shims. A shim, like the k8s-shim, might run in the same container or node but there is no guarantee it will. We also do not know the number of shims that will be used with one core scheduler. There is also still the possibility to have multiple instances of the same shim with one core scheduler.
+
+Shim configuration must be independent of the core scheduler configuration.
+## Scheduler Configuration
+Scheduler configuration covers all the configuration needed to start the scheduler and the dependent services. The configuration consists of a simple key value pair. All configuration to start the service must be part of this configuration.
+The scheduler configuration must exclude the queue related configuration.
+
+Scheduler configuration as currently identified
+* Bind host
+* Service port
+* Web bind host
+* Web service port
+* SSL config
+* Shims Configured
+* SchedulerACL
+
+Configuration to consider:
+* Assign multiple containers in one go: use case is bin packing, don’t spread an application over large number of nodes. Needs to become configurable.
+* Pre-emption related configuration:
+ * threshold: do not pre-empt from a queue if the cluster load is below a certain threshold.
+ * Interval: pause between pre-emption checks
+## Queue Configuration
+### Queue Definition
+On startup the scheduler will load the configuration for the queues from the provided configuration file after initialising the service. If there is no queue configuration provided the scheduler should start up with a simple default configuration which performs a well documented default behaviour.
+Based on the kubernetes definition this configuration could be a configMap <sup id="s1">[1](#f1)</sup> but not a CRD.
+
+The queue configuration is dynamic. Changing the queue configuration must not require a scheduler restart.
+Changes should be allowed by either calling the GO based API, the REST based API or by updating the configuration file. Changes made through the API must be persisted in the configuration file. Making changes through an API is not a high priority requirement and could be postponed to a later release.
+
+The queue configuration defines queues in a hierarchy: a tree. The base of the tree is the _root_ queue. The queue configuration must define a single _root_ queue. All queues that are defined in queue configuration are considered _managed_ queues.
+
+The root queue reflect the whole cluster. Resource settings on the root queue are not allowed. The resources available to the root queue are calculated based on the registered node resources in the cluster. If resources would be specified on the root limit the cluster would either be artificially limited to a specific size or expect resources to be available that are not there.
+
+Queues in the hierarchy in the tree are separated by the “.” dot character (ASCII 0x2E). This indirectly means that a queue name itself cannot contain a dot as it interferes with the hierarchy separator. Any queue name in the configuration that contains a dot will cause the configuration to be considered invalid. However we must allow placement rules to create a queue with a dot based input.
+
+Not all queues can be used to submit an application to. Applications can only be submitted to a queue which does not have a queue below it. These queues are defined as the _leaf_ queues of the tree. Queues that are not a _leaf_ and thus can contain other queues or child queues are considered _parent_ queues.
+
+Each queue must have exactly one _parent_ queue, besides the root queue. The root queue cannot have a _parent_ and will be automatically defined as a _parent_ queue type.
+A fully qualified queue name, case insensitive, must be unique in the hierarchy. A queue in the hierarchy can thus be only uniquely identified by its fully qualified path. This means that a queue with the same name is allowed at a different point in the hierarchy.
+Example:
+```
+root.companyA.development
+root.companyB.development
+root.production.companyA
+```
+In the example the queues _companyA_ and _companyB_ are _parent_ queues. Both _development_ queues are _leaf_ queues.
+The second instance of the _companyA_ queue is a _leaf_ queue which is not related to the first instance as it is defined at a different level in the hierarchy.
+
+The queue as defined in the configuration will be assigned a queue type. This can either be implicit based on how the queue is defined in the hierarchy or explicit by setting the optional _parent_ property as part of the queue definition. By default all queues will be assigned their type based on the configuration. There is only one case in which this should automatic process would need to be overridden and that is to mark a _leaf_ in the configuration as a _parent_. The use case is part of the [placement rules](#placement-rules-definition). In that case the configuration could be used to define a _parent_ queue for only _unmanaged_ queues.
+
+Access control lists provide a split between submission permission and administration permissions. Submission access to a queue allows an application to be submitted to the queue by the users or groups specified. The administration permissions allows submission to the queue plus the administrative actions. Administrative actions are currently limited to killing an application and moving an application to a different queue.
+
+Access control lists are checked recursively up to the root of the tree starting at the lowest point in the tree. In other words when the access control list of a queue does not allow access the parent queue is checked. The checks are repeated all the way up to the root of the queues.
+
+On each queue, except the root queue, the following properties can be set:
+* QueueType:
+ * Parent (boolean)
+* Resource settings:
+ * Guaranteed (resource)
+ * Maximum (resource)
+* Running Application limit:
+ * Maximum (integer)
+* Queue Permissions:
+ * SubmitACL (ACL)
+ * AdminACL (ACL)
+* Pre emption setting:
+ * PreEmptionAllowed (boolean)
+* Application sort algorithm:
+ * ApplicationSortPolicy (enumeration: fair, fifo)
+
+On the root queue only the following properties can be set:
+* Running Application limit:
+ * Maximum (integer)
+* Queue Permissions:
+ * SubmitACL (ACL)
+ * AdminACL (ACL)
+* Application sort algorithm:
+ * ApplicationSortPolicy (enumeration: fair, fifo)
+
+### User definition
+Applications are run by a user could run in one or more queues. The queues can have limits set on the resources that can be used. This does not limit the amount of resources that can be used by the user in the cluster.
+
+From an administrative perspective setting a limit of the resources that can be used by a specific user can be important. In this case a user is broadly defined as the identity that submits the application. This can be a service or a person, from a scheduling perspective there is no difference.
+User limits can prevent a take over of a queue or the cluster by a misbehaving user or application. From a multi tenancy perspective user limits also allows for sharing or subdivision of resources within the tenancy however that is defined.
+
+Adding user based limits will allow the cluster administrators to control the cluster wide resource usage of a user:
+* Running Application limit:
+ * Maximum (integer)
+* Resource setting:
+ * Maximum (resource)
+
+### Placement Rules definition
+Schedulers can place an application in a queue dynamically. This means that an application when submitted does not have to include a queue to run in.
+
+A placement rule will use the application details to place the application in the queue. The outcome of running a placement rule will be a fully qualified queue or a `fail`, which means execute the next rule in the list. Rules will be executed in the order that they are defined.
+
+During the evaluation of the rule the result could be a queue name that contains a dot. This is especially true for user and group names which are POSIX compliant. When a rule generates a partial queue name that contains a dot it must be replaced as it is the separator in the hierarchy. The replacement text will be `_dot_`
+
+The first rule that matches, i.e. returns a fully qualified queue name, will halt the execution of the rules. If the application is not placed at the end of the list of rules the application will be rejected. Rules can return queues that are not defined in the configuration only if the rule allows creation of queues.
+
+These queues created by the placement rules are considered _unmanaged_ queues as they are not managed by the administrator in the configuration. An administrator cannot influence the _unmanaged_ queue creation or deletion. The scheduler creates the queue when it is needed and removes the queue automatically when it is no longer used.
+
+Rules provide a fully qualified queue name as the result. To allow for deeper nesting of queues the parent of the queue can be set as part of the rule evaluation. The rule definition should allow a fixed configured fully qualified parent to be specified or it can call a second rule to generate the parent queue. By default a queue is generated as a child of the root queue.
+
+Example:
+Placing an application submitted by the user _user1_ whom is a member of the groups _user1_ and _companyA_ in a queue based on UserName:
+```
+Rule name: UserName
+ Parent: root.fixedparent
+Result: root.fixedparent.user1
+
+Rule name: UserName
+ Parent: SecondaryGroup
+ Filter:
+ Type: allow
+ Groups: company.*
+Result: root.companyA.user1
+
+Rule name: UserName
+Filter:
+ Users: user2,user3
+Result: denied placement
+```
+The default behaviour for placing an application in a queue, which would do the same as using the queue that is provided during submit, would be a rule that takes the provided queue with the create flag set to false.
+
+Access permissions will be enforced as part of the rule evaluation. For _managed_ queues this means that the ACL for the queue itself is checked. For an _unmanaged_ queue the parent queue ACL is the one that is checked. For the definition of the access control list and checks see the [Access Control Lists](#access-control-lists) chapter.
+
+Defining placement rules in the configuration requires the following information per rule:
+* Name:
+ * Name (string)
+* Parent
+ * Parent (string)
+* Create Flag:
+ * Create (boolean)
+* Filter:
+ * A regular expression or list of users/groups to apply the rule to.
+
+The filter can be used to allow the rule to be used (default behaviour) or deny the rule to be used. User or groups matching the filter will be either allowed or denied.
+The filter is defined as follow:
+* Type:
+ * Type (string) which can have no value (empty) or "allow" or "deny", case insensitive.
+* Users:
+ * A list of zero or more user names. If the list is exactly one long it will be interpreted as a regular expression.
+* Groups:
+ * A list of zero or more group names. If the list is exactly one long it will be interpreted as a regular expression.
+
+Proposed rules for placing applications would be:
+* Provided: returns the queue provided during the submission
+* UserName: returns the user name
+* PrimaryGroupName: returns the primary group of the user
+* SecondaryGroupName: returns the first secondary group of the user that matches
+* Fixed: returns the queue name configured in the rule
+* ApplicationType: returns the application type (if available)
+
+For _unmanaged_ queues in the current revision of the configuration you cannot provide any queue specific properties. However in the future we should consider propagating specific resource related settings from a _managed_ parent to the _unmanaged_ child, specifically:
+* Dynamic Resource settings:
+ * Guaranteed (resource)
+ * Maximum (resource)
+* Dynamic Running Application limit:
+ * Maximum (integer)
+
+### Configuration updates
+Updating the queue definition will allow updating the existing queue properties as well as adding and removing queues. A new queue definition will only become active if the configuration can be parsed. The change of the definition is an atomic change which applies all modification in one action.
+
+Updating the queue properties will not automatically trigger further action. This means that if the maximum number of resources of a queue or its parent is changed we leave the applications in the queue running as they are. The scheduler will adhere to the new property values which should see the convergence over time.
+
+A _managed_ queue will only be removed if it is removed from the configuration. Before we can remove a queue it must not be running applications. This means that when a _managed_ queue is removed from the configuration it must be empty or the system needs to allow the queue to drain. Forcing a _managed_ queue to be empty before we can remove it is not possible which means that _managed_ queues are removed in multiple steps:
+1. The queue is removed from the configuration
+1. The queue is marked as `draining`
+1. All managed queues that are `draining` and empty are removed
+
+Long running applications should be handled gracefully when removing a _managed_ queue. The scheduler should at least track and expose that a queue has been in a _draining_ state for an extended period of time. In the optimal case the application should be notified of the queue change to allow it to release resources. In all cases the queue administrators should be notified to allow them to take action. This action would currently be a manual move of the application to a different queue by the administrators.
+
+_Unmanaged_ queues that are not defined in the queue definition are created by the scheduler automatically based on the placement rules. _Unmanaged_ queues have a lifespan independent of the configuration. Whenever an _unmanaged_ queue is empty it will get removed. The queue will automatically be created again when a new application is requesting it via triggering the placement rule.
+
+Removing an empty _managed_ or _unmanaged_ queue is handled by the same removal code which must run independent of the configuration updates and scheduling actions.
+
+Configurations can change over time. The impact of a fail over or restart must still be investigated.
+Base point to make: a changed configuration should not impact the currently running applications. Queues that no longer exist should be handled somehow.
+
+### Access Control Lists
+The scheduler ACL is independent of the queue ACLs. A scheduler administrator is not by default allowed to submit an application or administer the queues in the system.
+
+All ACL types should use the same definition pattern. We should allow at least POSIX user and group names which uses the portable filename character set <sup id="s2">[2](#f2)</sup>. However we should take into account that we could have domain specifiers based on the environment that the system runs in (@ sign as per HADOOP-12751).
+
+By default access control is enabled and access is denied. The only special case is for the core scheduler which automatically adds the system user, the scheduler process owner, to the scheduler ACL. The scheduler process owner is allowed to make sure that the process owner can use the API to call any administrative actions.
+
+Access control lists give access to the users and groups that have been specified in the list. They do not provide the possibility to explicitly remove or deny access to the users and groups specified in the list.
+
+The access control list is defined as:
+```
+ACL ::= “*” | userlist [ “ “ grouplist ]
+userlist ::= “” | user { “,” user }
+grouplist ::= “” | group { “,” group }
+```
+
+This definition specifies a wildcard of * which results in access for everyone. If the user list is empty and the group list is empty nobody will have access. This deny all ACL has two possible representations:
+* an empty access control list.
+* a single space.
+
+If there is no access control list is configured access is denied by default.
+## Shim Configuration
+The shim configuration is highly dependent on the shim implementation. The k8s shim differs from the YARN shim. Currently the k8s shim is configured via command line options but we should not depend on that.
+
+### K8s shim
+The full configuration of the K8s shim is still under development.
+
+### YARN shim
+The full configuration of the YARN shim is still under development.
+
+---
+<br/><b id="f1"></b>1: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#should-i-use-a-configmap-or-a-custom-resource. [↩](#s1)
+<br/><b id="f2"></b>2: The set of characters from which portable filenames are constructed. [↩](#s2)
+<br/>`A B C D E F G H I J K L M N O P Q R S T U V W X Y Z a b c d e f g h i j k l m n o p q r s t u v w x y z 0 1 2 3 4 5 6 7 8 9 . _ -`
diff --git a/versioned_docs/version-1.5.0/design/scheduler_plugin.md b/versioned_docs/version-1.5.0/design/scheduler_plugin.md
new file mode 100644
index 0000000..f26b86a
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/scheduler_plugin.md
@@ -0,0 +1,112 @@
+---
+id: scheduler_plugin
+title: K8s Scheduler Plugin
+---
+
+<!--
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+-->
+
+## Background
+
+YuniKorn (on Kubernetes) is traditionally implemented as a ground-up implementation of a Kubernetes scheduler.
+This has allowed us to innovate rapidly, but is not without its problems; we currently have numerous places
+where we call into non-public K8S source code APIs with varying levels of (code) stability, requiring
+sometimes very disruptive code changes when we switch to new Kubernetes releases.
+
+Ideally, we should be able to take advantage of enhancements to new Kubernetes releases automatically.
+Using the plugin model enables us to enhance the Kubernetes scheduling logic with YuniKorn features.
+This also helps keep YuniKorn compatible with new Kubernetes releases with minimal effort.
+
+Additionally, it is desirable in many cases to allow non-batch workloads to bypass the YuniKorn scheduling
+functionality and use default scheduling logic. However, we have no way to do that today as the default
+scheduling functionality is not present in the YuniKorn scheduler binary.
+
+Since Kubernetes 1.19, the Kubernetes project has created a stable API for the
+[Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/),
+which allows plugins to be created which implement various extension points. Plugins implement one or more
+of these extension points, and are then compiled into a scheduler binary which contains the default
+scheduler and plugin code, configured to call into the plugins during normal scheduling flow.
+
+## Design
+
+We have added a scheduler plugin to the k8s-shim codebase which can be used to build a Kubernetes
+scheduler binary that includes YuniKorn functionality as well as the default scheduler functionality,
+significantly improving the compatibility of YuniKorn with upstream Kubernetes and allowing deployment of
+YuniKorn as the sole scheduler in a cluster with much greater confidence.
+
+Separate docker images are created for the scheduler. The traditional YuniKorn scheduler is built as
+`scheduler-{version}` while the new plugin version is built as `scheduler-plugin-{version}`. Either can be
+deployed interchangeably into a Kubernetes cluster with the same helm charts by customizing the scheduler
+image to deploy.
+
+## Entrypoints
+
+The existing shim `main()` method has been relocated to `pkg/cmd/shim/main.go`, and a new `main()` method
+under `pkg/cmd/schedulerplugin/main.go` has be created. This method instantiates the default Kubernetes
+scheduler and adds YuniKorn to it as a set of plugins. It also modifies the default scheduler CLI argument
+parsing to add YuniKorn-specific options. When the YuniKorn plugin is created, it will launch an instance
+of the existing shim / core schedulers in the background, sync all informers, and start the normal YuniKorn
+scheduling loop.
+
+## Shim Scheduler Changes
+
+In order to cooperate with the default scheduler, the shim needs to operate slightly differently when in
+plugin mode. These differences include:
+
+ - In `postTaskAllocated()`, we don’t actually bind the Pod or Volumes, as this is the responsibility of
+ the default scheduler framework. Instead, we track the Node that YK allocated for the Node in an
+ internal map, dispatch a new BindTaskEvent, and record a `QuotaApproved` event on the Pod.
+ - In `postTaskBound()`, we update the Pod’s state to `QuotaApproved` as this will cause the default scheduler
+ to re-evaluate the pod for scheduling (more on this below).
+ - In the scheduler cache, we track pending and in-progress pod allocations, and remove them if a pod is
+ removed from the cache.
+
+## Plugin Implementation
+
+To expose the entirety of YuniKorn functionality, we implement three of the Scheduling Framework Plugins:
+
+### PreFilter
+
+PreFilter plugins are passed a reference to a Pod and return either `Success` or `Unschedulable`, depending
+on whether that pod should be considered for scheduling.
+
+For the YuniKorn implementation, we first check the Pod to see if we have an associated `applicationId`
+defined. If not, we immediately return `Success`, which allows us to delegate to the default scheduler for
+non-batch workloads.
+
+If an `applicationId` is present, then we determine if there is a pending pod allocation (meaning the
+YuniKorn core has already decided to allocate the pod). If so, we return `Success`, otherwise `Unschedulable`.
+Additionally, if an in-progress allocation is detected (indicating that we have previously attempted to
+schedule this pod), we trigger a `RejectTask` event for the YuniKorn core so that the pod will be sent back
+for scheduling later.
+
+### Filter
+
+Filter plugins are used to filter out nodes that cannot run a Pod. Only Pods which pass the PreFilter stage
+are evaluated.
+
+For the YuniKorn plugin, we follow similar logic to PreFilter, except that we also validate that the pending
+pod allocation matches the node YuniKorn chose for the pod. If the node matches, we transition the pending
+allocation to an in-progress allocation. This helps ensure that we stay in sync with the default scheduler,
+as it is possible that we allow an allocation to proceed but the bind fails for some reason.
+
+### PostBind
+
+The PostBind extension point is used informationally to notify the plugin that a pod was successfully bound.
+
+The YuniKorn implementation uses this to clean up the outstanding in-progress pod allocations.
diff --git a/versioned_docs/version-1.5.0/design/simple_preemptor.md b/versioned_docs/version-1.5.0/design/simple_preemptor.md
new file mode 100644
index 0000000..7dbce18
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/simple_preemptor.md
@@ -0,0 +1,114 @@
+---
+id: simple_preemptor
+title: DaemonSet Scheduling using Simple Preemptor
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+# Design & Implementation of Preemption for DaemonSet Pods using Simple Preemptor
+
+The simplistic approach to preempt or free up resources of running applications for DaemonSet pods. A good example of daemonset pod is fluentd logging pod which is very essential for any applicaton pod running in the node for logging.
+
+## When to start preemption?
+[YUNIKORN-1184](https://issues.apache.org/jira/browse/YUNIKORN-1184) ensures daemon set pods have been allocated properly if resources are available on the required node, otherwise, reserve the same required node so that it can be picked up to make reservation as reserved allocation (AllocatedReserved) in the next scheduling cycle. However, the whole process of modifying the reservation to reserved allocation depends on how much resources are freed up in the meantime. Duration for freeing up resources by natural course is non deterministic as it depends on the running pods execution on that particular node and it could take slightly longer time as well.
+
+By any chance, before the next run of the regular scheduling cycle (context#schedule() ), resources become available and particularly on that specific required node, then nothing needs to be done. It just moves ahead with the next steps. In case of resource constraints, unlike the regular pod reservation, other nodes cannot be tried by calling application#tryNodesNoReserve() as this demonset pod needs to run only on the specific required node. Instead, we can fork a new go routine (trigger_preempt_workflow() ) to run the workflow steps described below.
+
+
+## How to do preemption?
+K8s does the preemption based on the pod priority. Pods with lower priority would be chosen first and so on. Proposal is not to depend on k8s for Preemption. Instead, Core should take the responsibility of finding out the list of pods that needs to be preempted, making communications to Shim and finally expecting the preempted resources to allocate to the corresponding daemonset automatically as part of the regular scheduling cycle.
+
+### Steps in trigger_preempt_workflow() go routine:
+
+##### Reservation age check (1)
+We can introduce a new Reservation age “createtime” (can be added to the reservation object) to check against the configured value of preemption_start_delay, a property to define the minimal waiting time to start the preemption process. Once reservation age exceeds this waiting time, the next step would be carried out. Otherwise, the corresponding reservation has to wait and can be processed next time.
+
+##### Get allocations from specific required Node (2)
+Get all allocations from the required node of the daemonset pod and go through the below Pre-filter pods step to filter the pods not suited for a preemption.
+
+##### Pre-filter pods to choose Victims/Candidates suited for Preemption (3)
+
+Core should filter the pods based on the following criteria:
+
+###### DaemonSet Pods
+
+All Daemonset pods should be filtered out completely irrespective of priority settings. Depending on the “requiredNode” value of pod spec, these pods can be filtered out and cannot be taken forward for the remaining process.
+
+
+
+##### Ordering Victim pods (4)
+
+###### Pods classification
+
+Once pods has been filtered out, need to classify the pods based on its types:
+
+1. Regular/Normal Pods (RP)
+2. Driver/Owner Pods (DP)
+3. Preemption Opt out Pods (OP)
+
+This classification ensures different treatment for each type of pod so that victims can be chosen among these pods in the same order. Please refer to the above diagram. It shows the 2-Dimensional array (NOTE: “Array” has been used only for documentation purposes, need to finalize the appropriate data structure) with each sub array holding pods of the same type. 1st sub array has RP’s, 2nd sub array has DP’s, 3rd sub array has OP’s and goes on.
+
+Regular/Normal Pods (RP)
+
+The regular/normal pods should be gathered and placed in the 1st sub array as these pods are given first preference for choosing the victims. In general, preempting these pods have very little impact when compared to other types/classes of pods. Hence, keeping these pods in the first subarray is the right choice
+
+Application Owner (DP)
+
+Pod acting as owner/master for other pods in the same application should be placed in the 2nd sub array because preempting those kinds of pods has a major impact when compared to Regular pods. We can select these pods by checking whether any owner reference exists between this pod and other pods. This will help prevent scenarios such as a driver pod being evicted at a very early stage when other alternatives are available for choosing the victim.
+
+Preemption Opt out (OP)
+
+Pods can be allowed to run with the Preempt opt out option. So, Pods marked with opt out should be placed in the 3rd sub array and can be used to choose victims as a last option. For now, we can use a label such as yunikorn.apache.org/allow-preemption: false for detecting those pods.
+
+
+As and when we want to introduce a new class/type of Pods, a new sub array would be created for the same and would be placed in the main array based on its significance.
+
+###### Sorting Pods
+
+Each sub array should be sorted based on the multiple criteria:
+
+1. Priority
+2. Age
+3. Resource
+
+Each sub array would be sorted priority wise, age wise and finally resource wise. The 1st sub array carrying Regular Pods has 4 pods of priority 1 and 2 pods of Priority 2. Among the 4 pods of the same priority, 3 pods are of the same age as well. Hence, again sorting resource wise really adds value and sorts them in the above shown order. Please refer to “zone”.
+
+#### Victim pods selection strategy (5)
+
+Introduce a new configuration, preemption_victim_poselection_strategy with different options (single, multiple etc) but later options act as fallback to earlier one. Defining an order for these options should be possible and upto the Administrator to place the options in an order he/she desires. Depending on the values, the whole selection strategy mechanism can decide whether a “fallback” approach among these options should be followed or not. Depending on the value, the selection strategy process would continue with the next option only if no selection has been made with the earlier option or stop only with the first option even if no selection has been made.
+
+##### 1. Single Victim Pod
+
+Single Victim Pod, but resource deviation between victim pod and daemonset pod is not beyond configurable percentage. Configuring deviation with lower percentage (for example, 5% or 10%) helps prevent evicting victim pods already running with higher resource requirements. If there are many single victims found within the defined deviation %, then selection starts based on deviation % ascending order as intent is to choose the victim as close as possible to the daemonset pod resource requirements.
+
+##### 2. Multiple Victim Pods
+
+Multiple Victim Pods, but no. of victim pods not more than configured value. This selection strategy helps to choose more than one victim, starts with the victim (resource wise descending order) and goes upto to a stage where total resource of victims meets the daemonset resource requirements but ensuring total count of victim pods not exceeding configured value.
+
+New config: preemption_victim_pods_selection_strategy
+Possible values are single,multiple (default) or multiple,single or single or multiple
+
+In case of more than one value (for ex. single,multiple), fallback would be followed as described above.
+
+#### Communicate the Pod Preemption to Shim (6)
+
+Once the list of pods has been finalized for preemption, Core can make a call to Shim for termination using notifyRMAllocationReleased (with type as TerminationType_PREEMPTED_BY_SCHEDULER). Shim can process the request as usual by making a call to K8s to delete the pod and subsequently call failTaskPodWithReasonAndMsg to notify the pod with reasons.
+
+### What happens after Preemption?
+
+Shim makes a call to K8s to delete the pod. Once k8s delete the pod, shim gets a notification from k8 and passes the information to core. This flow happens for any pod deletion and exists even today. So, even for preempted resources, we can leave it upto the regular scheduling cycle and Core-Shim communications to allocate these freed up preempted resources to the daemonset pod as node has been already reserved much earlier before the above described whole preemption workflow has begun.
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/design/state_aware_scheduling.md b/versioned_docs/version-1.5.0/design/state_aware_scheduling.md
new file mode 100644
index 0000000..f92f93c
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/state_aware_scheduling.md
@@ -0,0 +1,112 @@
+---
+id: state_aware_scheduling
+title: Batch Workloads Ordering with StateAware Policy
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+## The problem
+A common pattern while processing data is that the application can be divided into multiple stages.
+Another way to look at this is the fact that processing needs to be kicked off and that the first step is to start a driver or manager for the application.
+Later stages might depend on the previous stages.
+When running applications in a size limited environment this could lead to a resource exhaustion when submitting multiple applications at the same time.
+These first stages might consume all available resources leaving no room for the next stage(s) to start.
+Often this issue is caused by having a high number of applications start simultaneous and trying to get resources in parallel.
+### Example issue
+When submitting numerous Spark applications in a short amount of time the drivers will all be started shortly after each other.
+The drivers consume all available resources in a queue or in the whole cluster.
+After starting the drivers they will request resources for the executors.
+Since the queue or cluster has no resources left the executors will not be started.
+The driver cannot progress.
+The only way that progress would be made is if and when one of the drivers finishes or fails and frees up resources for executors to be started.
+
+## Design
+### Design goals
+1. Prevent resource exhaustion by first stage allocations
+1. Improve chance for jobs to get minimal required resources over others
+
+### None goals
+1. This is NOT an implementation of Gang scheduling.
+1. No change to the currently supported FAIR or FIFO scheduling algorithms
+1. Fix resource quota usage outside of the core scheduler for submitted but waiting applications
+
+### Possible solutions
+Other batch schedulers like the YARN schedulers use a limit on the number of simultaneous running applications.
+They use either resource constraints on the driver or management stage or set a hard limit of the number of applications that can run in a queue.
+The draw back of that solution is that it does not work well in a cluster that can scale up or down automatically in a cloud environment.
+To work around that percentage based limits could be set on the consumed resources for the driver or management stage.
+This does not alleviate the fact that driver or management stages can be of any size, large and or small, which complicates the percentage scenario further as it does not give a predictable behaviour.
+
+A different solution would be to assume a specific behaviour of the applications.
+Using that assumption a limit on the applications could be set based on the state it is in.
+The spark driver and executor behaviour is the most usual use case.
+This would provide a way to limit scheduling to existing applications and only drip feed new applications into the list of applications to schedule when there are resources available.
+
+### Algorithm
+The algorithm described here is based on the drip feed of new applications into the applications to schedule based on the states of all applications.
+Scheduling is based on the applications in a queue.
+The algorithm will be applied at a queue level.
+This is not a cluster wide setup.
+
+What we want to achieve is the following behaviour: only schedule one (1) application that is in its early stage(s) (called a starting state) at the same time.
+Only consider another new application if and when the previous application has transitioned out of the starting state.
+Applications will always be allocated resources on a first in first out basis based on submission time.
+That means that an application that is newly added and in its starting phase will only get resources if applications in the later stages do not need any resources.
+
+This algorithm will be implemented as an application sorting policy on a queue.
+This allows specific queues to limit parallel application startup while other queues with different work loads can schedule without or with different limitations.
+
+### Fallback mechanism
+A fallback mechanism has to be built into the algorithm.
+Not all applications will request more than one allocation.
+The other case that has to be accounted for could be a misbehaving or a slow application.
+Having an application stuck in the starting state could cause a scheduler livelock and starvation of other applications.
+
+The fall back mechanism proposed is as simple as a time limit on the starting state.
+This means that any application auto progresses out of the starting state.
+The time limit will be set to five (5) minutes hard coded as a starting point and will not be made configurable.
+
+The other fallback mechanism considered was making the number of allocations for the starting state configurable.
+This option provided a number of issues which made it difficult to implement.
+One of the main stumbling blocks is the fact that it requires the application submitter to specify the value.
+It also does not guarantee that the application will leave the starting state either and does not fix the livelock issue.
+If an application was submitted with five required allocation but due to issues during the run never asked for more than four then the livelock would still occur.
+
+Setting a default of zero (0) would also not work as it would bypass the starting state.
+It would make the sorting policy an opt-in instead of an opt-out.
+Setting a default of one (1) does not give us much enhancement to what we currently propose.
+It makes the sorting policy an opt-out but does not give the cluster administrator any control over the scheduling behaviour.
+Weighing those against each other the proposal is to not make this configurable.
+
+### Example run
+Using Spark applications as an example: a new application can only be scheduled if the previous application has at least one (1) executor allocated.
+
+
+
+Assume we have the following Spark apps: App1 & App2 as in the diagram above. The applications were submitted in that order: App1 first, then App2. They were both submitted to the same queue.
+
+1. Both applications are in the queue waiting for the first allocation: accepted by the scheduler. App1 has requested driver D1 and App2 has requested driver D2.
+1. The scheduler sorts the application and allows 1 accepted application to be scheduled (no starting applications yet): App1 as the oldest applications with an outstanding request is scheduled.
+App1 is allocated its driver (D1) and progresses to starting.
+App2 request for a driver is ignored as the scheduler is starting App1 (only 1 application in starting or accepted state is scheduled).
+1. App1 requests executors E11 and E12. The scheduler assigns E11 and E12. At this point the application state changes to running when it has at least 1 executor allocated.
+1. App2 has been waiting to get the driver allocated. Since there are no applications in a starting state the scheduler looks at App2 which is in an accepted state. App2 moves from the accepted state to starting when the driver is allocated.
+1. App2 requests its executor E21. The application state changes to running when E21 is allocated.
+
+This process would repeat itself for any new application submitted.
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/design/user_group_enforcement.md b/versioned_docs/version-1.5.0/design/user_group_enforcement.md
new file mode 100644
index 0000000..4b3c888
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/user_group_enforcement.md
@@ -0,0 +1,478 @@
+---
+id: user_group_resource_usage_enforcement
+title: User Based Quota Enforcement
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Enforcing resource allocation usage is currently limited to a queue.
+As part of the allocation process for an application, which owns the allocation, triggers an update of the resource usage for users and groups.
+The tracking for [user resource usage tracking in YuniKorn](user_based_resource_usage_tracking) is described and implemented independently of the enforcement.
+
+This design builds on top of the tracking design as referenced above. Like a queue a user/group could have a limit set.
+A limit can specify a resource usage quota, and an application usage limit.
+Enforcing the limit which connects the usage to the limit settings is covered in this design.
+
+## Goals
+
+* Implementation of the enforcement for user and group limits:
+ * Resource usage quota
+ * Running applications
+* Configuration processing
+ * Process limit object values from the queue configuration
+* Updates to the tracking interface to support enforcement as part of increases:
+ * Increase a user/group running applications
+ * Increase a user/group resource usage
+
+## Non Goals
+
+* User and group retrieval is part of the k8shim which is out of scope
+* Exposing usage as a metric into prometheus
+* Enforcement of queue quotas or running applications.
+* Tracking of users and or groups, see [User and Group usage tracking design doc](user_based_resource_usage_tracking)
+
+## Configuration processing
+
+The queue configuration definition has a limit object defined as [per the documentation](../user_guide/queue_config#limits).
+User and group limits are based on those objects.
+However the current configuration processing only performs a syntax check not a content check.
+
+### Generic behaviour
+
+Limits can be configured for queues at any level. Limits are applied recursively in the case of a queue limit.
+This means that a limit on the root queue is an overall limit in the cluster for the user or group.
+A root queue limit is thus also equivalent with the partition limit. While if both the root queue limit and the partition limit exist and are not equal, an error will be returned.
+
+A limit is per individual user or group. It is not a combined limit for all users or groups specified in the limit object.
+See the below limit object as an example.
+In this example both users sue and bob are allowed to run 2 applications and can each use 10 cpu and 250G of memory.
+Users and or groups not defined are not affected by this limit object.
+
+```yaml
+- limit: "example entry"
+ maxapplications: 2
+ maxresources:
+ cpu: 10
+ memory: 250G
+ users:
+ - sue
+ - bob
+```
+
+In the case that limits are configured at multiple levels in the hierarchy a fixed order of enforcement is used.
+Like with queue quotas, checks start at the leaf queue and work their way up the hierarchy finishing at the root level.
+For the limit configuration processing the same needs to be implemented and the config should be rejected if it does not comply.
+Any limit set at any level below the root must always be smaller or equal to the root limit.
+
+The checks for the limit configuration must be part of the generic configuration checks currently performed as part of the loading of the configuration.
+We cannot allow the configuration to be accepted at the queue level and later be denied at the limit level checks. A change is all or nothing, we cannot apply partial changes. Configuration checks must be made without impacting the scheduling cycles. After the change has passed the configuration check the changed configuration must then be applied in a single action.
+
+### Wildcard interpretation
+
+Wildcard can only be used as the only entry in the limit object according to the documentation.
+There can be multiple limit objects in the overall limits for a queue.
+As part of this design the interpretation of the wildcard needs to be defined.
+Wildcards interpretation is related to the ordering of the limit objects in the overall limits object.
+The usage and interpretation will be different for users and groups.
+
+In general terms: allowing a wildcard in the user or group list only as part of the last entry of the limits list.
+After the wildcard has been added for either the user or group list we do not allow a user or group list with a non wildcard entry.
+This is especially important for the group resolution. It does allow specifying a user and group wildcard with different limits set.
+In all cases, there will only be a match using the wildcard if none of the earlier limit entries match.
+
+#### User
+
+Users that are specified as part of a limit will be assigned that specific limit.
+If a limit with a wildcard is specified all users that are not already assigned a limit will be assigned the wildcard limit.
+For the user entry the wildcard sets a default quota for every user on that queue. Overridden by the named user entries.
+
+Specifying a single limit for users using the wildcard is allowed. There is no requirement to have a wildcard entry.
+
+#### Group
+
+For groups the interpretation is slightly more complex.
+As explained as part of the tracking documentation the group that the usage is tracked against must match the membership of the user.
+A wildcard group matches any group.
+
+Specifying a wildcard for the group limit sets a cumulative limit for all users in that queue.
+If there is no specific group mentioned the wildcard group limit would thus be the same as the queue limit.
+For that reason we do not allow specifying only one group limit that is using the wildcard.
+There must be at least one limit with a group name defined.
+
+The combination of one or more named group limits with a wildcard group limit is allowed.
+There is no requirement to have a wildcard entry.
+
+The hierarchy is considered bottom up, starting from the leaf up to the root.
+The first group that is specified in the queue hierarchy for limiting usage that matches a group in the _UserGroup_ object is selected for that application.
+That definition has an impact on the wildcard interpretation and the wildcard has an impact on the group selection.
+The wildcard will indirectly affect group quotas set for a specific group higher up in the hierarchy.
+This point needs clear documentation as part of the group selection algorithm and its side effects.
+
+### Example configuration
+
+An example of a set of limits. A mixture of a user and multiple groups each with a limit.
+Followed by the catch-all for both users and groups with separate resource limits.
+
+In the example below all users, except the user “sue” have the same limit set. Members of the group “development” and the group “test” both have the same limit set. All other users who are not a member of the group “development” or “test” are counted towards the cumulative group limit.
+```yaml
+limits:
+- limit: "specific user"
+ users:
+ - "sue"
+ maxresources: {memory: 25G, vcore: 5}
+- limit: "specific groups"
+ groups:
+ - "development"
+ - "test"
+ maxresources: {memory: 100G, vcore: 10}
+- limit: "user catch all"
+ users:
+ - "*"
+ maxresources: {memory: 10G, vcore: 1}
+- limit: "group catch all"
+ groups:
+ - "*"
+ maxresources: {memory: 50G, vcore: 10}
+```
+
+### Syntax checking
+
+As part of the already existing queue configuration processing the syntax for the configuration is checked.
+This check is triggered also for a reload of the file, via the config map, or the REST interface.
+Rudimentary limit object checking is implemented as part of the config check.
+
+The above described changes around the wildcards and what is allowed is not part of the configuration validation and must be added.
+
+### Quota checks
+
+No checks are performed for the limits against the queue quotas.
+This means that as part of the current checks a limit can be set that is higher than the queue would allow.
+There is also no hierarchical check in place. This means that a child could set higher limits than is possible based on the parent settings.
+
+The configuration validation processing must be extended to the limit objects.
+At each level in the configuration the resource quota assigned in a limit must be smaller than the maximum allowed at that level for the queue.
+The queue quota processing already has this builtin and the limit should be checked against the quota.
+
+The hierarchical check for limits contains a further complexity.
+A limit at one level might not apply to the same users or groups as the limit at the level below it.
+An implementation decision will need to be made on how far we want to go with the checks.
+Most, if not all, of the configurations that we do not want to allow do not break the system.
+We will not be able to check user and group interactions as we do not know the group memberships.
+The only thing we can check is the same user or group at different levels in the hierarchy.
+
+## Configuration updates and storage
+
+The configuration is read as part of the queue configuration.
+The queue configuration when processed creates, deletes or updates queues from the queue structure as appropriate.
+For limits linked to users or groups we could pre-generate the named entries but we cannot do that for wildcards.
+To support wildcards we need to at least store the configuration inside the user group manager code.
+
+The definition of the configuration store and the methods that need to be defined will be left as an implementation detail.
+It must be internal to the user group manager and will be based on the existing configuration objects.
+
+The configuration for the limits should be maintained as a separate hierarchy from the scheduling queues and the tracked users and groups.
+This is required to allow parallel processing of changes and scheduling.
+This becomes more important with an increased number of users, groups and or queues.
+
+Proposal is to pass the full configuration after it has been accepted by the system to an independent update routine for updating the affected objects in the user group manager.
+
+### Processing changes existing objects
+
+As configuration changes are based on queue changes a single queue change could trigger multiple tracker object changes.
+
+Applying the change from the configuration to an existing object in the structure of the user group manager would be initiated from the queue configuration traversal.
+All entries inside the limit are then processed.
+A comparison between the existing and new values is required to allow the correct removal of a limit via the updated configuration.
+
+If the user object does not exist the update will be a NOP, otherwise the user object will be updated by calling the setter method with the corresponding values.
+
+Wildcards will trigger an update of all objects of the specific type, user or group, for the queue they are linked to.
+These kinds of updates could be expensive which also warrants the case for comparison of the existing values.
+
+### Queue tracking
+
+The queueTracker object is the location where the limits are actually stored. The user and group objects do not contain the settings.
+
+The setter methods defined need to transparently handle the update.
+There should be no difference in behaviour to an update of existing values or setting values that do not exist.
+As an example: a leaf queue might have a value set for the maximum resources for the user.
+The new configuration does not have a value set. The setter must clear the existing value.
+Second example: a limit on a queue is not set for a group.
+The new configuration does set a limit for the group.
+This should not result in the queue(s) in the hierarchy to be created.
+If the queue exists in the hierarchy it will be updated.
+
+The following change is made to the queueTracker object.
+New fields are added to store the limit for the applications and the resources.
+New methods are added to allow updating these new values from a configuration change.
+
+```
+package user_group_manager
+
+type queueTracker struct {
+ queueName string
+ resourceUsage *Resource
+ runningApplications map[string]boolean
+ maxResourceUsage *Resource
+ maxRunningApps unit64
+
+ childQueues map[string]*queueTracker
+}
+
+(qt *queueTracker) setMaxApplications(count uint64, queuePath string)
+(qt *queueTracker) setMaxResources(resource *Resource, queuePath string)
+```
+
+An empty queue path indicates that the change needs to be applied to the current queue object. The queuePath is a fully qualified queue path that starts with “root’.
+
+### User tracking
+
+The following change is made to the userTracker object. New methods to allow updating the new limit values stored in the queue objects.
+```
+package user_group_manager
+
+(ut *userTracker) setMaxApplications(count uint64, queuePath string)
+(ut *userTracker) setMaxResources(resource *Resource, queuePath string)
+```
+
+### Group tracking
+The following change is made to the groupTracker object. New methods to allow updating the new limit values stored in the queue objects.
+
+```
+package user_group_manager
+
+(ut *groupTracker) setMaxApplications(count uint64, queuePath string)
+(ut *groupTracker) setMaxResources(resource *Resource, queuePath string)
+```
+
+## New queue to track
+
+As part of the usage tracking new objects are created for the users and or groups that are tracked.
+For users there is a change required to make sure the correct limits are set.
+This happens at two points while processing users.
+First is during the creation of a new user tracker object.
+Second is while increasing the usage of a queue which does not exist yet in the hierarchy.
+
+The logic to retrieve the configuration should be built into the creation of the queueTracker objects.
+The objects are not pre-created. They are only created as part of the _increaseResource_ call.
+The current call does not allow passing in the identity that the queue object is tracking for.
+That detail is stored in the container object: the userTracker or groupTracker object.
+
+To allow the automatic creation of the queueTracker objects the identity will need to be passed into the increaseResource call of the queueTracker object.
+The identity and type are required to find the configuration setting and apply the proper limits during the creation of the queueTracker object.
+
+```
+package user_group_manager
+
+type trackingType int
+
+const (
+ none trackingType = iota
+ user
+ group
+)
+
+(qt *queueTracker) increaseResource(queuePath, applicationID, identity string, type trackingType, usage *Resource)
+```
+
+The _trackingType_ specifies one of the two types we can track. An iota is preferred above the usage of a plain integer or boolean (user true or false) kind of setup.
+
+## Enforcement changes
+
+### Tracker interface update
+
+The current tracker does not support denying a change. For enforcement of quotas on the usage there has to be a possibility to deny a change.
+This denial can only occur on an increase of tracked resources. A decrease of the usage, i.e. a removal of an allocation because a pod has finished, can never be denied.
+
+This limits the change to support quota enforcement to a single change in the Tracker interface.
+The _IncreaseTrackedResource_ call returns the state of the increase requested.
+The return value can either be true or false. The call must only return true if the increase is allowed for both the user and the group tracked for the request.
+If either of the userTracker or groupTracker quota would be exceeded no change is made to either object and false is returned.
+
+The following change is made to the Tracker interface:
+
+```
+package user_group_manager
+
+type Tracker interface {
+ IncreaseTrackedResource(queuePath, applicationID string, usage *Resource, user *security.UserGroup) bool
+ DecreaseTrackedResource(queuePath, applicationID string, usage *Resource, removeApp bool, user *security.UserGroup)
+}
+```
+
+The change must be applied to both userTracker and groupTracker.
+Until the change is applied to both the userTracker and groupTracker, no other updates are allowed of either object.
+The increase and decrease processing must still follow the same order. Both must update the userTracker and related groupTracker objects in the same order.
+This was a requirement described in the tracking design and that has not changed.
+
+If the call to _IncreaseTrackedResource_ returns false the allocation that triggered the change needs to be abandoned.
+Any change that was made to objects inside the scheduler, like an application, queue or node, must be reversed.
+The scheduling cycle proceeds as if no allocation was made.
+
+The error return value for both the _IncreaseTrackedResource_ and the _DecreaseTrackedResource_ have been removed.
+The reason is that there is nothing that could be done by the caller to change the outcome or fail any changes if an error was returned.
+Both functions should still log the error case as it could point to a bug in the code.
+A caller should never have to pass in empty or nil parameters.
+
+### User tracking
+
+The following change is made to the userTracker method. There is no change to the structure itself for the enforcement checks:
+
+```
+package user_group_manager
+
+(ut *userTracker) increaseResource(queuePath, applicationID string, alloc *Resource) bool
+```
+### Group tracking
+
+The following change is made to the groupTracker method. There is no change to the structure itself for the enforcement checks:
+```
+package user_group_manager
+
+(gt *groupTracker) increaseResource(queuePath, applicationID string, alloc *Resource) bool
+```
+
+### Queue tracking
+
+The following change is made to the queueTracker method. There is no change to the structure itself for the enforcement checks:
+```
+package user_group_manager
+
+(qt *queueTracker) increaseResource(queuePath, applicationID, identity string, type trackingType, usage *Resource) bool
+```
+
+Note that the above change includes the change described earlier which was made to support the automatic creation of the queue objects as part of the increaseResource call.
+### Hierarchy traversal
+
+The implementation of the hierarchical check should follow the same pattern as we currently do for the queues.
+There is one major difference between the queue checks and the user and group checks.
+For the queues we do not attempt to allocate unless there is unused quota available.
+This starts at the top of the hierarchy and prevents recursing into the depth of the hierarchy towards the leaf queue.
+
+For the users and groups we have already ended up as the leaf of the structure.
+Checks will thus start at the leaf and work their way up to the root of the structure.
+A check will be performed while traversing up the hierarchy. The traversal up the tree will stop if the check fails.
+The change is applied when unwinding the traversal. This will guarantee a change is only committed when the whole hierarchy can be updated.
+
+
+
+Diagram of the quota check traversal related to the queue hierarchy.
+
+## Exposure of quota details
+
+The usage tracking information that is part of the user group manager is exposed for external consumption via REST.
+The quotas set as part of the queue objects should be exposed in the REST output.
+Exposing the quota independent of the usage makes interpretation difficult.
+
+The quota set in the queueTracker at the specific entry is the quota that is enforced on that entry.
+From a troubleshooting perspective this would be a requirement to allow an administrator to assess state.
+
+Based on the current REST api definition the data exposed in the following two endpoints will be updated to expose the limits:
+_/ws/v1/partition/{partitionName}/usage/users_
+_/ws/v1/partition/{partitionName}/usage/groups_
+
+For both endpoints we expose the full queue hierarchy. As an example below the approximate output for the users endpoint for one user:
+
+```json
+[
+ {
+ "userName": "user1",
+ "groups": {
+ "app1": "tester"
+ },
+ "queues": {
+ "queuename": "root",
+ "resourceUsage": {
+ "memory": 12000000000,
+ "vcore": 12000
+ },
+ "runningApplications": ["app1", "app2"],
+ "children": [
+ {
+ "queuename": "root.default",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app1"],
+ "children": [],
+ "maxApplications": 10,
+ "maxResources": {
+ "memory": 50000000000,
+ "vcore": 100000
+ }
+ },
+ {
+ "queuename": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app2"],
+ "children": [],
+ "maxApplications": 0,
+ "maxResources": {}
+ }
+ ],
+ "maxApplications": 10,
+ "maxResources": {}
+ }
+ }
+]
+```
+
+An example below the approximate output for the groups endpoint for one group:
+```json
+[
+ {
+ "groupName" : "tester",
+ "users": ["user1"],
+ "queues":
+ {
+ "queuename": "root",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app2"],
+ "children": [
+ {
+ "queuename": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app2"],
+ "children": [],
+ "maxApplications": 2,
+ "maxResources": {
+ "vcore": 10000
+ }
+ }
+ ],
+ "maxApplications": 0,
+ "maxResources": {}
+ }
+ }
+]
+```
+
diff --git a/versioned_docs/version-1.5.0/design/user_group_handling.md b/versioned_docs/version-1.5.0/design/user_group_handling.md
new file mode 100644
index 0000000..57994fa
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/user_group_handling.md
@@ -0,0 +1,290 @@
+---
+id: user_group
+title: User/Group handling and lookup design
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Currently, user-handling in YuniKorn is handled in a relatively simple manner. If the submitted pod contains a user label the content will be set as the username. The label name is configurable and defaults to yunikorn.apache.org/username. The value is extracted in the shim code [GetUserFromPod](https://github.com/apache/yunikorn-k8shim/blob/v1.0.0/pkg/common/utils/utils.go#L220) and passed to the core. If the label is not present or no value is set the username will be set to default.
+
+However, this makes it possible for every authenticated user to freely impersonate anyone. This allows a user to circumvent quotas and other user-based restrictions inside YuniKorn. Therefore, we need a robust, secure way to make sure that users are who they say they are. This impersonation should not be confused with [Kubernetes impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation).
+
+In the core part of YuniKorn, we also perform group resolution, similar to a unix based system or for instance Hadoop. The default implementation is a nil-lookup. It simply populates the group slice with a single entry which is the same as the username. This is not fit for purpose and was only added as a placeholder until proper user and group resolution could be added.
+
+## Goals
+
+- Define a stable way for user & group retrieval from Kubernetes
+- Assess fit of the current group resolution in the shim
+- Definition of user and group info on Kubernetes objects
+
+## Non Goals
+
+- Authorisation or Authentication as part of:
+ - YuniKorn
+ - Kubernetes
+- Changes to the way users are propagated between the shim and the core
+
+
+## Kubernetes user and group handling
+
+Dealing with how users are actually managed in Kubernetes is out of scope. What is important is the definitions that are used by Kubernetes for users. In Kubernetes, there are two kinds of accounts defined: [service and user accounts](https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/). User accounts are for humans. Service accounts are set up for processes which run inside a pod and want to access the API server.
+
+### User model
+
+Kubernetes, at this point in time, only manages service accounts, not users. Kubernetes does not have objects which represent user accounts. A user with valid credentials is considered authenticated.
+
+During the authentication phase the user account is mapped to a service account. For authorisation Kubernertes uses a sophisticated role-based (RBAC) or a simpler attribute-based (ABAC) model. All authorisation is linked to the service accounts, not to the user accounts. Beside the authentication there is thus no need for the user accounts. This is reflected in Kubernetes as after the authentication phase the user account is no longer tracked or available.
+
+The model used by Kubernetes thus limits what we can do to retrieve the user and or group details. All objects that are available inside the cluster to a scheduler, like YuniKorn, only have the service account information available. Getting access to the user account means that we need to intercept the information at an earlier stage.
+
+### User information as part of authentication
+
+User accounts are required during the authentication stage. Authentication in Kubernetes is configurable via plugins. For more details, see the official document on [Kubernetes authentication](https://kubernetes.io/docs/reference/access-authn-authz/authentication/). The list of supported authentication plugins, not all of these plugins will be configured and or active by default:
+- X.509 certificate
+- Static token file
+- OpenID connect tokens
+- Authentication proxy
+- Authentication webhooks ([LDAP example](https://itnext.io/implementing-ldap-authentication-for-kubernetes-732178ec2155))
+- Bootstrap token
+- Service Account Tokens
+
+
+When using test clusters created by Minikube or KIND, X509 authentication is used by default and the current context is updated with the appropriate certificates and keys, making cluster access transparent.
+
+Authentication, independent of which option is used, provides the following details:
+- Username
+- userID
+- Groups
+- Extra information
+
+Where the details are stored or how they are retrieved depends on the authentication plugin used.
+
+Kubernetes defines a special user for anonymous access, the `system:anonymous` user belonging to the `system:unauthenticated` group. Anonymous access is a security issue and should have been turned off in most clusters. This default mapping to a standardised user does provide the possibility to process authenticated and anonymous workloads in the same way.
+
+### Groups in authentication
+
+Users may belong to zero or more groups. The authentication system of Kubernetes adds one of two default groups to each user. The `system:authenticated` group is added to the list of groups for authenticated users, the `system:unauthenticated` group is added for anonymous users.
+
+For example, when generating an X509 certificate, we can define the username and organisation information like this:
+```
+openssl req -new -key alice.key \
+-out alice.csr \
+-subj "/CN=alice/O=users/O=devops"
+```
+This certificate, after being processed, provides an authenticated user with the username alice and sets the groups list to users, devops and system:authenticated. Other authentication methods allow specifying groups in their specific way. How groups are specified for each authentication option is not relevant to this design as it is encapsulated in the authentication layer and not in the exposed model.
+
+### Current findings and challenges
+
+None of the objects in Kubernetes contain the authentication information and thus the user details. After the authentication has been processed the objects inside Kubernetes either use the service account that was specified as part of the object definition or the Default service account.
+
+
+That means that the pod which is processed by YuniKorn, represented by the `k8s.io/api/core/v1/Pod` object, only has the service account as a reference.This is similar to the details available to someone using the Kubernetes tools to describe the object.
+
+The only way to retrieve the authentication information is during the admission phase of a new object into the Kubernetes system. This process is handled by the admission controllers which are part of the decision process to allow or disallow the new object to be created, modified or removed.
+
+This finding does allow YuniKorn, via its admission controller, to retrieve the authentication information of objects.
+
+
+
+
+However there are some further complications to this process. Pods are the units that the scheduler works with. Objects like deployments or jobs do not get scheduled, they in turn create a pod to be scheduled. Pods could thus be created in a number of ways.
+
+If a pod is created by a direct request, then the pod creation process contains the authentication information. This allows the YuniKorn admission controller to access and propagate the authenticated user information as part of the MutatingAdmissionWebhook.
+
+An example of a review of a direct call to create a pod with the user information shown:
+```
+INFO webhook/admission_controller.go:180 AdmissionReview {"Namespace": "default", "UID": "3f8c8df5-0272-4c85-874f-0aa1ece74fbc", "Operation": "CREATE", "UserInfo": {"username":"minikube-user","groups":["system:masters","system:authenticated"]}}
+```
+
+On the other hand, if a deployment, job or replica set is submitted, then the pod is not directly created. This also means that the user is not identifiable. The pod that YuniKorn, via the admission controller, sees is created by a controller. The user in the authentication information is not the end user that created the original object but the controller service account.
+
+An example review of a pod created as part of a Kubernetes job with the user information shown:
+
+```
+INFO webhook/admission_controller.go:177 AdmissionReview {"Namespace": "default", "UID": "be94ea46-c1d2-442a-a46a-60589d582abd", "Operation": "CREATE", "UserInfo": {"username":"system:serviceaccount:kube-system:job-controller","uid":"32abc062-eb53-4dca-9669-55251d687939","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]}}
+```
+It is possible to create separate webhooks for the API objects that could result in the creation of pods. Other objects that we should watch and mutate if necessary:
+- Deployment
+- ReplicaSet
+- DaemonSet
+- StatefulSet
+- Job
+- CronJob
+- ReplicationController (deprecated)
+
+For example, adding a modification on the admission controller which instals a MutatingAdmissionWebhook for deployments, the username can be determined:
+
+```
+INFO webhook/admission_controller.go:483 Deployment validation - pass through {"User": {"username":"minikube-user","groups":["system:masters","system:authenticated"]}}
+```
+
+By intercepting the original object the submitting user information can be retrieved in the admission controller. The set of objects of interest, mentioned in the list above, all contain a pod template to specify the pod that will be created by the controller. As part of the mutating admission webhook the pod template can be updated with the user information.
+
+To confirm that this approach is viable a simple proof of concept (POC) was created. The example POC code for the current admission controller adds the currently defined yunikorn.apache.org/username label to the pod template of a deployment: [Github commit](https://github.com/pbacsko/incubator-yunikorn-k8shim/commit/478d688675a1ca98afb895672295d61b47172d19)
+
+There is an interaction with the processing of a directly created pod. Pods that are created via a controller will have the user information already set via the pod template. The admission controller processing the request should allow for the user information to be present and not overwrite or replace it with the controller service account. This case will be further defined and detailed in the chapter on [Providing user and group externally](#providing-user-and-group-externally) due to the similarity in solution design.
+
+## YuniKorn group retrieval
+
+As mentioned in the introduction, the YuniKorn core can retrieve a group for a given user by doing a lookup (see implementation in [usergroup.go](https://github.com/apache/yunikorn-core/blob/161c38100317ec239965e21806ebb35c6583ddcd/pkg/common/security/usergroup.go)). The lookup is pluggable and the current implementations provided are a simple OS-based and a dummy lookup.
+
+The dummy lookup mimics the simple unix OS behaviour for new users. It sets the group to the same value as the username. The dummy lookup is enabled as the default lookup.
+
+Resolving users using the OS-based resolver has limited functionality. The container that the scheduler runs in has only a base OS installed. There is no integration with an external identity provider. That means resolving groups for users running pods would normally fail as they do not exist on the OS layer that is accessible from the scheduler.
+
+The YuniKorn core only performs a group lookup if the UserGroupInformation object passed from the shim to the core does not define a list of groups. This is independent of the resolver defined in the code. By default the shim in its current implementation only passes the user name to the core. That means it always triggers the group resolution.
+
+For our purposes, both options are too simplistic. YuniKorn core should not perform any group retrieval but instead rely on the shim and the group and user information retrieved there. With the current functionality to only resolve if no group details are provided changes required should be minimal. The current default dummy resolver should be updated to not set any group and just leave the current groups. Even if that means the user is not part of any group. Based on the [Groups in authentication](#groups-in-authentication) details when using the Kubernetes shim we should always have at least one group set. All users are members of either system:authenticated or system:unauthenticated. Even without changes the current group retrieval code could be used as is.
+
+The design does not require any code changes in the core code.
+
+## Proposed user information object
+
+The admission controller has access to more detailed user information than what is available on the objects that are available to the shim. See the [Current findings and challenges](#current-findings-and-challenges) for more details.
+
+
+First change is that we need to use annotations instead of [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/). The value of labels is limited to 63 characters, which will not be flexible enough to hold group information. If a user is a member of multiple groups it could easily exceed the value limit. Spitting the groups over multiple labels or encoding is not a viable option as stability is not a guarantee.
+
+Supporting a single label for the user’s name and annotations for the groups is not a good user experience. We must provide a single object that can contain the user name as well as the group details.
+
+### Annotation name
+
+The new name for the annotation should clearly cover the content and not interfere with existing annotations. Beside that we are free to use any name that complies with the naming [conventions for names](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set).
+
+The proposal is to use one annotation:
+`yunikorn.apache.org/user.info`
+
+### Annotation content
+
+There is already a definition for the user information as part of the scheduler interface.The definition in the scheduler interface is:
+
+```
+message UserGroupInformation {
+// the user name
+string user = 1;
+// the list of groups of the user, can be empty
+repeated string groups = 2;
+}
+```
+
+The content of the `UserGroupInformation` message as defined in the Scheduler Interface provides a stable Go object with defined group ordering. Since the content of an annotation must be a simple string we should use a simple json representation for the user info that contains the user name and a list of groups. This also guarantees the ordering of the groups to be stable and will not change when read from the pod.
+
+
+An example of the annotation for a user called joe who is a member of the group developers. The system:authenticated group is added by the Kubernetes authentication layer:
+
+```json
+yunikorn.apache.org/user.info = "
+{
+ username: \"joe\",
+ groups: [
+ \"developers\",
+ \"system:authenticated\"
+ ]
+}"
+```
+
+### Annotation mutation
+
+In the current design a restriction on mutating the annotation is required to prevent users from updating the pod after creation. After YuniKorn is informed of the pod’s existence and the application is created the user information will not change inside YuniKorn. Kubernetes however does not guarantee that the shim will be informed of the pod creation before any modifications are made to the pod. This would leave a short window for a user to create a pod and then update the pod. Allowing a user to overwrite the annotation must be prevented to make sure we have guaranteed and auditable user information.
+
+The admission controller must therefore validate update requests that include changes to the annotation. It is the only way to guarantee that the user information in the annotations cannot be modified.
+
+The YuniKorn admission controller is currently only registered as a mutating admission controller for pods with the operation type “create”. This can be extended to include operation type “update” for pods. A validating hook is registered for the configmap object, this one could be extended to include the pod object.
+
+This means that there are two options for the update operation:
+- mutating
+- validating
+
+In the validating case we can simply reject the change if the annotation is changed as part of the update request. In the mutating case we can undo the change by overwriting the annotation with the original value. Both solutions will work, and will take about the same amount of processing time in the admission controller.
+
+The proposal is to add the pod object to the validating webhook for update operations.
+
+### Backwards compatibility
+
+The current label will remain supported for the 1.x minor releases. Deprecation will be announced with the first release that supports the new annotation. Messages mentioning the processing of the old label will also be logged at a WARN level in the logs. The existing behaviour might require changes to the default setting see [Configuration](#configuration) for more details.
+
+Removal of processing of the currently supported label should be part of the next major release. The next major release is 2.0.0. This is based on the fact that we do not have a deprecation policy defined as yet.
+
+Preference in processing will be with the new annotations. In the case that both the user label and new annotations are present on the namespace the new annotation will be used. Using both the user label and new annotations, i.e. merging of the two, will not be supported.
+
+## Providing user and group externally
+
+As with the current label for the username we could allow for providing user and group information via annotations on the submitted objects. Although this is not a secure solution, as is pointed out in the introduction, support for this option might be a requirement to support certain use cases.
+
+We must build in an option to turn the functionality off and or mitigations to limit the possibility to bypass reading the authentication information. Based on the earlier diagram of the admission controller being used to intercept the authentication information a similar diagram for this use case. The admission controller will validate the annotation provided by the user, or system, and enforce compliance with the rules set by the administrator.
+
+
+
+
+### Pods from controllers
+
+The first use case for allowing the user information to be provided on pods when they are created is linked to controller created pods.
+
+To correctly account for these pods the admission controller should be able to not overwrite the user information if the submitting user is one of a specific group of users. Depending on the way the system is configured all controllers run as the same service account or each controller uses a specifically created service account. The [RBAC documentation](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#controller-roles) describes the two options.
+
+If the controller manager runs with the option `--use-service-account-credentials` each controller uses a service account in the kube-system namespace. Running with the option set is recommended from an auditing point of view.
+
+If the option is not enabled, which seems to be the default, the controller submits the pods as the user `system:kube-controller-manager`.
+
+Both values of the option must be supported by the admission controller. That means the admission controller must have an allow list which contains at least the following users:
+- `system:serviceaccount:kube-system:*`
+- `system:kube-controller-manager`
+
+An option would be to further limit the service accounts to a subset of the service account from the kube-system namespace. The configuration should support this without changes, it would come down to a change of the default value for the allow list.
+
+There is only a limited set of controllers that can create pods. The group information does, at this point in time, not look deterministic enough to warrant a group allow list.
+
+Note that the user system:kube-controller-manager name is based on some older github issue references and needs to be confirmed in a test setup.
+
+### External applications
+
+In a similar setup applications could be used to provide a frontend for pod creation. These front-ends might or might not use individual users to submit pods. They could even leverage jobs or other objects to indirectly create pods.
+
+As an example an application might be used to provide a simple user interface to create a Spark application. Apache Airflow, Livy or similar tools come to mind.
+
+The admission controller should allow the annotation to be provided as part of the pod creation. This follows the same code and configuration path as designed for the pods submitted by the controllers.
+
+This functionality must be configurable separately from the pods created by the controllers. A similar allow list should be configurable to limit the user. For this use case we also should support an allow list for groups of users. This should allow administrators to specify one or more groups of users that are allowed to set the user information on the pods.
+
+For the first design there are no plans to add deny lists for either groups or users.
+
+## Configuration
+
+As described in the previous sections, making the behaviour configurable is a requirement. The proposal is to add the following configuration values for the behaviour. All configuration options are prefixed by “ADMISSION_CONTROLLER_”.
+
+
+The setting should be migrated to a configuration object or file when that is implemented for the admission controller. At this point the configuration uses environment settings as part of the deployment file.
+
+
+|Name|Description|Default|
+|--|--|--|
+|`BYPASS_AUTH`|Allow external users to create pods with user information set. Type: boolean |false
+|`BYPASS_CONTROLLERS`|Allow controller users to create pods with user information set. Type: boolean |true
+|`SYSTEM_USERS`|Regular expression for the allowed controller service account list. Type: string|"system:serviceaccount:kube-system:*"
+|`EXTERNAL_USERS`|Regular expression for the allowed external user list. Type: string|""
+|`EXTERNAL_GROUPS`|Regular expression for the allowed external group list. Type: string|""
+
+The configuration will be read as part of the startup of the admission controller. Automatic refresh or reload of the settings to allow changes while the admission controller is running is not part of the design as it is not supported for any other settings at this moment.
+
+If `BYPASS_AUTH` is set to true the admission controller will not add the annotation to a pod if the annotation is not present and the deprecated user labell is set. If the annotation is not set and the user label is not set the new annotation will be added. In the case that `BYPASS_AUTH` is false, the default, the admission controller will always add the new annotation, regardless of the existence of the deprecated label.
+
+The behaviour will take into account the user and group filters as defined in `EXTERNAL_USERS` and `EXTERNAL_GROUPS`. This means that retaining the existing behaviour and preferencing the existing label will require changing a default setting.
diff --git a/versioned_docs/version-1.5.0/design/user_group_manager.md b/versioned_docs/version-1.5.0/design/user_group_manager.md
new file mode 100644
index 0000000..e70cad9
--- /dev/null
+++ b/versioned_docs/version-1.5.0/design/user_group_manager.md
@@ -0,0 +1,410 @@
+---
+id: user_based_resource_usage_tracking
+title: User Based Resource Usage Tracking
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Introduction
+
+Tracking resource allocation usage is currently limited to a queue. An application, which owns the allocation, is the source of that resource usage. All applications that are part of the queue will count towards the resource usage of the queue. A queue could also have a resource usage quota set. The queue’s resource quota is enforced during the scheduling cycle.
+
+An application is always owned by a user/group. One user/group could own applications in one or more queues. Like a queue a user/group could have a limit set. A limit can specify a resource usage quota, and an application usage limit. Tracking these usages per user/group is currently not implemented and is covered as part of this design.
+
+## Goals
+
+- Specify tracking usage in relation to the following objects:
+ - users
+ - applications
+ - queues
+- An interface to define how to track running applications:
+ - Read a user/group running applications
+ - Increase a user/group running applications *)
+ - Decrease a user/group running applications
+
+- An interface to define how to track usage:
+ - Read a user/group resource usage
+ - Increase a user/group resource usage *)
+ - Decrease a user/group resource usage
+
+ *) This call could be extended for enforcement later.
+
+- Expose user/group tracked information through REST API
+
+## Non Goals
+- User and group retrieval is part of the k8shim which is out of scope
+- Exposing usage as a metric into prometheus
+- Specify enforcement rules
+- Implementation of enforcement
+- Interface definition to allow enforcement
+
+## Tracking definitions
+
+### User vs Group tracking
+
+Tracking is based on the _security.UserGroup_ object defined in the core scheduler. User and group information is retrieved by the shim and communicated via the _UserGroupInformation_ object as defined in the scheduler interface. A _security.UserGroup_ object contains:
+- user (exactly 1)
+- groups (0 or more)
+
+Limits can be configured for both users and groups at the same time in the same queue. So, in the case of users for which no limit has been configured explicitly but group (to which the user belongs to) limits have configured works based on the group limits configuration. In case of users for which a limit has been configured explicitly works based on its own configured limits irrespective of whether configuration has been set for the corresponding group (to which the user belongs to) or not. In other words, user limit settings always have the higher precedence when group limit settings have been configured too.
+
+"*" is also allowed for both users and groups limit configuration. In this case, any user or group needs to go through enforcement checks to ensure limit settings are being honoured properly. However, the tracker does not need to track the wildcard user as a specific user or group is already being tracked during their activities.
+
+### Group limitations
+
+Users can belong to zero, one or multiple groups. In some systems, like unix, a concept of a primary group exists. This concept is not universal and there is no consistent definition of a primary group either. If the system does not define a primary group how would we distinguish between the groups?
+
+In the case the user is a member of multiple groups the assignment of usage to a group also becomes problematic. The decision which group to assign the usage to becomes arbitrary. As an example consider a user with multiple groups assigned. Joe has groups dev and test assigned. When an application is started does that application count towards the limits for the dev group or against the limits for the test group or both?
+
+The _UserGroup_ object contains a slice, list or array, of groups which has a fixed order from the moment of creation. The group resolution configured might not guarantee an ordering which means we cannot rely on the order in the _UserGroup_ to be stable. A stable and pre-defined way of choosing a group to track usage against must be defined.
+
+The first point to define is that tracking can only occur against zero (0) or one (1) group. Usage can never be tracked against multiple groups. If a user is a member of multiple groups, a group is selected when the first usage is registered for the application. The group selected out of the list of groups in the _UserGroup_ object to track against is based on the hierarchical queue limit configuration. The hierarchy is considered bottom up, starting from the leaf up to the root. The first group that is specified in the queue hierarchy for limiting usage that matches a group in the _UserGroup_ object is selected for that application.
+
+If no group limits are defined, in the hierarchy, usage is not tracked for a group. If no groups are defined in the _UserGroup_ object no usage will be tracked against a group.
+
+The tracker considers only the group received as part of the _UserGroup_ object at the time of first usage registration for the application in the tracker. In case of group changes while the user’s application is actively being tracked, the tracker does not consider the group change. However, If the user submits a new application, the most current group information at that point in time will be used.
+
+Until there is full support for groups in the k8shim, group tracking will be limited. No groups are passed by the k8shim to the core. This results in the _UserGroup_ object in the current implementation to always have only one group. The group name will be the same as the username. We also have excluded configuration processing for the tracking. For the current tracking implementation as described in the document these limitations will result in the user and group linked to the user to track the same usage.
+
+### Cluster or queue based tracking
+
+Tracking user usage will be based on the usage per queue. The full hierarchy of the queue will be tracked. We should not just track the leaf or root of the hierarchy. The overall cluster usage is the same as the usage of the root queue. Cluster level usage does not need separate tracking.
+
+The overhead of tracking per queue should be really limited. In most cases a user will run their workloads in a specific queue. Deeply nested queues do also add overhead to the tracking. During testing the impact of workload spread and or deep nesting needs to be assessed. If needed this can be called out in the documentation.
+
+We also need to take into account that multiple members of a group could be using different queues in the hierarchy. If tracking usage would be limited to the leaf level, aggregating group usage becomes more difficult and loses transparency.
+
+The documentation around limits, part of the enforcement, reference different limits at different levels for users or groups. Tracking at just the leaf or root level could break that functionality. The design should support the current documented features. If during enforcement design a different decision is made then the tracking is not the deciding factor.
+
+## Application tracking
+
+Applications go through a number of states within the core scheduler. Allocations linked to applications are the triggers for application state changes. From a tracking perspective we need to define what is considered a running application.
+
+Since placeholders and placeholder timeout can play a role in state changes the implementation of the running application tracking must take into account these cases:
+- no gang defined
+- gang defined, all or some placeholders replaced
+- gang style "soft" defined, all placeholders time out
+- gang style "hard" defined, all placeholders time out
+
+### Running state entry
+
+The application when submitted and placed into a queue is in the _New_ state. At this point there is no allocation or pending request present on the application. After one or more requests, _AllocationAsks_, are added the application moves into an _Accepted_ state. The _Accepted_ state is exited when the first _Allocation_ is added to the application. The application then transitions into the Starting state.
+
+At this point a resource quota would be used by the application and the application should be considered as running from a tracking perspective. This means that the addition of the first _Allocation_ onto the application also must be the trigger point for the increase of the running applications. This trigger point for tracking is when the application is in the _Accepted_ state. This is also the point at which the group for the usage tracking needs to be set as described in the [group limitations](#group-limitations).
+
+Note that currently, the application state transition code block in application_state.go updates the application running queue metrics when the application enters _Running_ state. The metric must be updated to be consistent with the above definition of a running application. Linking this back to a state transition the entry into the Starting state should be used.
+
+### Running state exit
+
+An application should be considered running as long as at least one allocation is assigned to the application. The removal of the last allocation should decrease the number of running applications. However, we also need to consider the pending requests.
+
+As part of the application states the application enters into a _Completing_ state when the last allocation is removed and there are no pending requests. This automatically progresses into the _Completed_ state if nothing changes for the application. Applications going into the _Completing_ state all have no allocations left and pending requests are considered. The entry of the _Completing_ state thus could be used as a point for triggering the decrease of the running application count.
+
+However there are circumstances that the application goes from the _Completing_ state back into a _Running_ state. This is linked to auto restart of a pod by Kubernetes. This case must be handled.
+
+A last case that needs to be considered is an application that fails. If an application fails it moves into the Failing state. The application does not enter _Completing_ but could have incremented the running applications. A failure could be triggered by the gang scheduling style "hard" or the removal of a partition.
+
+## Resource usage tracking
+
+Resource usage is tracked simply based on the addition or removal of an allocation.
+
+Removal of an allocation is an out-of-band action. The shim notifies the core scheduler that an allocation no longer exists and the core updates the tracking based on that removal. If an allocation is preempted the core initiates the removal but still gets notified by the shim that the removal has occurred. Until the confirmation of the shim is received no tracking updates must be created.
+
+The replacement of a placeholder by the real allocation for gang scheduling follows a similar concept. However, in that case there is a possibility that there is a discrepancy between the placeholder size and the real allocation. See [YUNIKORN-982](https://issues.apache.org/jira/browse/YUNIKORN-982) for more detailed information on this issue. The changes that fixed the accounting for the queues and nodes also need to be taken into account for user based resource tracking.
+
+Adding a new allocation to an application triggers the update of the tracked resources for the user and or group as described in the [group limitations](#group-limitations). Enforcement is not a goal of the design document but the enforcement implementation needs to be taken into account. The point to add the resource usage must line up with the point that the queue structure is currently updated and the queue quotas are enforced.
+
+The order of the updates: queue structure first then user or user first then queue structure is debatable. The current proposal is to implement the following:
+- Update user and group based usage
+- Queue quota enforcement and update
+- If the queue enforcement fails, roll back the user and group based update.
+
+## Scheduler transparency
+
+Resource usage tracking is based on the allocated resource, and linked to the application’s queue path and the application’s _user_ field. Application tracking is a simple count of the applications, and is linked to the application’s queue path and the application’s user field.
+As an extension on this we could consider adding the _applicationID_ into the tracking of the running applications to help with troubleshooting. This is part of the current proposal. The proposal is to not link the running application tracking to the applicationID.
+
+The user field of an application is the earlier mentioned _security.UserGroup_ object and has all the identifying details needed to track the usage.
+
+From the schedulers perspective there should be no special setup needed. The tracking should always work without the need to pre-create a user, group or queue based entry. This means that the tracking cannot fail even if the user or group has no enforcement linked to it and does not exist in the tracker.
+
+In summary the tracker must implement:
+- User and or group creation on the fly.
+- Auto cleanup of tracking information no longer needed.
+- Configuration processing needed for enforcement.
+
+## User Group Manager (UGM)
+
+Tracking all the resources will be part of a new self-contained module in the scheduler. This means that there must be no imports from the `scheduler.objects` package. This will allow us to run the tracker separately and push updates, if needed, into a background processing queue.
+
+There will be one user group manager created per partition as part of the partition creation process. There will be no tracking of user quotas over a partition boundary.
+
+The new self-contained module, user group manager contains all specifications and code for tracking and can be extended in the future to include enforcement code.
+
+Tracking the resources and application count for all users occurs from startup. This happens irrespective of whether enforcement is configured for the user or group. By starting tracking independently of the enforcement changes in enforcement will always have a consistent view of the resources tracked at least at a user level. Recovery must be tested in combination with the tracking code.
+
+Storing data in the user group manager is based on the _security.UserGroup_ structure content. The built-in _UserGroupCache_ resolution cannot guarantee that the same object is returned for the same user each time.
+
+### UserGroup cache impact
+
+The concept of a stable user entry in the cache is not currently present in the _UserGroupCache_. This concept will most likely also not be possible based on the fact that group resolution does not guarantee ordering. Based on the assumption that we cannot guarantee group ordering for all group resolutions that are possible the tracking will not rely on it.
+
+Currently, _UserGroupCache_ holds the _UserGroup_ object entry only for the cache period. This is either 300 seconds for a positive resolution or 30 seconds for a negative resolution. After this cache time the entry will be deleted as part of the cache cleanup/expiration process. If the same user submits a second application after the expiry, a new _UserGroup_ object will be created and stored in the cache. The _UserGroup_ object has a property "resolved", which holds a timestamp at which the user group resolution has been executed for the first time. The cache cleaner wakes up after every 60 seconds and deletes all entries whose "resolved" timestamp is older than the cache timeout.
+
+Other internal changes might be needed to make sure all _UserGroup_ objects are cached correctly and returned as expected internally in the cache. These are left as implementation details for the cache changes.
+
+### Tracking structure
+
+The user group Manager must be able to track usage for users and groups. Users are the primary access point for all tracking. The user group Manager structure will implement the [tracker interface](#tracker-interface).
+
+````
+package user_group_manager
+
+type Manager struct {
+ users map[string]*userTracker
+ groups map[string]*groupTracker
+
+ sync.RWMutex
+}
+(m *Manager) checkAppGroup(queuePath, applicationID string, user *security.UserGroup)
+````
+
+All objects in the user group manager will be managed internally. Creation of user and or group tracker objects is based on the request to track usage. The objects are added to one of the two structures in the user group manager. Adding user or group objects can only be triggered by an increase of tracked resources. The creation of the user and or group tracker objects is handled by the user group manager. The user group manager is also responsible for setting the usage information for the queue hierarchy directly after creation.
+
+The resolution of the group that the usage is tracked against for an application is handled by the user group Manager via the checkAppGroup. As part of an increase of resource usage the Manager will check if the application is already tracked for the user. If the application is already tracked nothing changes. If the application is not tracked the manager will resolve the group to track against and update the userTracker object. If there is no active tracker registered for the group a new groupTracker object will be created and initialised as required.
+
+Removal of user or group objects can only be triggered after a decrease of tracked resources has removed the last usage. Usage tracked for a user must be zero. For group tracker objects the application reference count must be zero.
+Removal of the user and or group tracker objects is done periodically. This cleanup must be `go routine` safe. Updates to either map could be triggered from different routines, scheduler allocation processing or a release of an allocation by a shim.
+
+### Tracker interface
+
+The interface describes the functions that will be exposed by the user group manager. The other scheduler components can rely on the functions defined in the interface to be available to increase, decrease applications and or resource usage and get the tracked resources.
+
+```
+package user_group_manager
+
+type Tracker interface {
+ GetUserResources(user *security.UserGroup) *Resource
+ GetGroupResources(group string) *Resource
+
+ GetUsersResources() []*userTracker
+ GetGroupsResources() []*groupTracker
+
+ IncreaseTrackedResource(queuePath, applicationID string, usage *Resource, user *security.UserGroup) error
+ DecreaseTrackedResource(queuePath, applicationID string, usage *Resource, removeApp bool, user *security.UserGroup) error
+}
+```
+
+The _IncreaseTrackedResource()_ and _DecreaseTrackedResource()_ calls will implement the tracking data updates. The Get* calls are providing safe access to the tracked data to be used as part of exposing the data in the REST interface. See [exposure of usage tracking](#exposure-of-usage-tracking) for more detail.
+
+Applications are added automatically during the call to increase the tracked resources. The first time the applicationID is encountered the structures are updated accordingly and tracking starts. The removal of the application is triggered by leaving the [running state](#running-state-exit), this is equivalent to the removal of the last usage and must also set the removeApp flag to true.
+
+Calling _IncreaseTrackedResource()_ and _DecreaseTrackedResource()_ with empty or nil parameters is not allowed and will return an error to be handled by the caller. The _GetUserResource_ and _GetGroupResource_ calls can return a _nil_ Resource value. The _GetUsersResource_ and _GetGroupsResource_ calls can return an empty slice.
+
+### User tracking
+
+The userTracker contains all tracked usage information for a user. Changes for all usage, even the group usage, are part of the change that gets triggered on the user object.
+
+The userTracker object contains a reference map to the usage for each application that is tracked linking it to the groupTracker object for the specific application. Multiple applications can be tracked against the same groupTracker object.
+
+The group references will be set for each application when usage is added for an application and is not mutable. The existence of a groupTracker for the application can be checked by calling _hasGroupForApp_ on the userTracker object. It returns true in case the application ID exists in the group mapping, false if it does not. The return value is only based on the existence of the applicationID as the key, not on the value it references.
+
+The groupTracker can be set in the userTracker object by calling _setGroupForApp_. If the applicationID already exists, the call to set the groupTracker reference is a noop, nothing will change. The reference cannot be changed after it is set. The groupTracker reference can be nil. In the case of a nil reference, no tracking for that application against a group is performed. Even a nil reference cannot be replaced.
+
+An increase or decrease of the resources tracked for the user requires the applicationID to be set to a non-empty value. On removal of the last allocation for the application the group reference will be removed from the userTracker object. To remove a running application from the userTracker the applicationID must be set in the _decreaseResource_ call and the flag removeApp must be set to true. Removal of the groupTracker reference can only occur as part of a _decreaseResource_ call.
+
+If there are no applications left in the group references the object will be removed by the cleanup run from the user group manager. This will also mean that there is no usage tracked anymore by the referenced queueTracker object.
+
+The usage per queue is stored in a queue structure specific for this user. The queueTracker object provides a hierarchical structure. This structure starts at the root of the queue hierarchy and only contains child queues for which a usage is being tracked. The hierarchy must be updated, add or delete of child entries, as part of the updates that are processed.
+
+The creation of the root queueTracker is part of the creating a new userTracker object.
+
+```
+package user_group_manager
+
+type userTracker struct {
+ userName string
+ group map[string]*groupTracker
+ queue *queueTracker
+
+ sync.RWMutex
+}
+
+newUserTracker(queuePath string, user *security.UserGroup) *userTracker
+
+(ut *userTracker) increaseResource(queuePath, applicationID string, usage *Resource)
+(ut *userTracker) decreaseResource(queuePath, applicationID string, usage *Resource, removeApp bool)
+(ut *userTracker) hasGroupForApp(applicationID string) bool
+(ut *userTracker) setGroupForApp(applicationID string, groupTrack *groupTracker)
+```
+
+### Group tracking
+
+The groupTracker contains all tracked usage information for a group. This object can be shared by multiple users for one or more applications. It adds the applications that are tracked for this specific group automatically to the list. For this it expects an applicationID to be set to a non-empty value for each _increaseResource_ call.
+
+To simply decrease the resources tracked for the group the applicationID could be empty. On removal of the last allocation for the application the application will be removed from the groupTracker object. To remove a running application from the groupTracker the applicationID must be set in the _decreaseResource_ call and the flag removeApp must be set to true.
+
+If there are no entries left in the applications tracked in the groupTracker the object will be removed by the cleanup run from the user group manager. This will also mean that there is no usage tracked any more by the referenced queueTracker object.
+
+The usage per queue is stored in a queue structure specific for this group. The queueTracker object provides a hierarchical structure. This structure starts at the root of the queue hierarchy and only contains child queues for which a usage is being tracked. The hierarchy must be updated, add or delete of child entries, as part of the updates that are processed.
+
+The creation of the root queueTracker is part of the creating a new groupTracker object.
+
+```
+package user_group_manager
+
+type groupTracker struct {
+ groupName string
+ applications map[string]bool
+ queue *queueTracker
+
+ sync.RWMutex
+}
+
+newGroupTracker(queuePath string, group string) *groupTracker
+
+(gt *groupTracker) increaseResource(queuePath, applicationID string, usage *Resource)
+(gt *groupTracker) decreaseResource(queuePath, applicationID string, usage *Resource, removeApp bool)
+```
+
+### Queue tracking
+
+The queueTracker acts as a base for both user and group tracking. It provides the actual location for the tracked usage.The top of the structure, which is referenced from the userTracker or groupTracker objects, contains the root queue usage. The queueTracker is only referenced by one userTracker or one groupTracker object and must not be shared. This means that we should not require any locking within the queueTracker object.
+
+The queue hierarchy below the root, for which a usage is registered, is part of the childQueues. Each part of the full queue path creates a new level of objects. This structure shows a sparse hierarchical queue structure. It may not represent the full queue structure as known in the scheduler.
+
+The creation of the root queueTracker returns a plain object without usage set and a fixed queueName of "root". The usage should then be updated by a call to _increaseResource_. The call must handle the recursive creation of the queue path as given. It expects an applicationID to be set to a non-empty value for each call.
+
+To simply decrease the resources tracked for the queue the applicationID can be empty. On removal of the last allocation for the application the application will be removed from the queueTracker. To remove a running application from the queueTracker the applicationID must be set in the _decreaseResource_ call and the flag removeApp must be set to true.
+
+```
+package user_group_manager
+
+type queueTracker struct {
+ queueName string
+ resourceUsage *Resource
+ runningApplications map[string]boolean
+
+ childQueues map[string]*queueTracker
+}
+
+newQueueTracker() *queueTracker
+
+(qt *queueTracker) increaseResource(queuePath, applicationID string, usage *Resource)
+(qt *queueTracker) decreaseResource(queuePath, applicationID string, usage *Resource, removeApp bool)
+```
+
+### Update consistency
+
+All changes are triggered against a userTracker object. Since enforcement of a quota is out of scope of this design we can ignore update failures due to quota exhaustion. There is no requirement or possibility that updates need to be rolled back.
+
+Usage updates can be triggered from two points: allocation by the scheduler and releases by a shim. Allocations as well as releases need to update both user and group usage information. We must guarantee that for both the full hierarchy is changed or neither are changed.
+
+The releases are also processed asynchronously from the scheduling cycle. This means that we have to be able to guarantee that the increases and decreases are applied consistently to both objects also. Locking of the objects to serialise changes must be in place to support this.
+
+The other point is the update ordering and locking. The increase and decrease processing must follow the same order. Both must update the userTracker and related groupTracker objects in the same order. This is a requirement to prevent the processing of being able to deadlock while updating the same user for a simultaneous increase and decrease.
+
+For groupTracker updates that originate from two different users the locking of the groupTracker object is required. One of the two updates will be able to proceed while the other waits for the update to finalise.
+
+A final consideration is the consistency of two updates on the same object that follow each other closely. If the first update removes all existing usage the userTracker or groupTracker object is cleared for removal. While the update is in progress a second update comes in to add a new usage to the same object. The resolution of the userTracker in the user group manager has finished. The code has a reference to the object. Processing is waiting to lock the userTracker object. This point could cause the object to become unlinked and usage tracking information to be lost. The implementation needs to prevent this race condition.
+
+### Exposure of usage tracking
+
+The usage tracking information that is part of the user group manager must be exposed for external consumption. All our current information is exposed via a REST call.
+
+Based on the current REST api definition for other object the proposal is to expose the following new endpoints:
+
+- /ws/v1/partition/{partitionName}/usage/users
+- /ws/v1/partition/{partitionName}/usage/groups
+
+For both endpoints we expose the full queue hierarchy. As an example below the approximate output for the user's endpoint for one user:
+
+```json
+[
+ {
+ "userName": "user1",
+ "groups": {
+ "app2": "tester"
+ },
+ "queues":
+ {
+ "queuename": "root",
+ "resourceUsage": {
+ "memory": 12000000000,
+ "vcore": 12000
+ },
+ "runningApplications": ["app1", "app2"],
+ "children": [
+ {
+ "queuename": "root.default",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app1"],
+ "children": []
+ },
+ {
+ "queuename": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": [
+ "app2"
+ ],
+ "children": []
+ }]
+ }
+ }
+]
+```
+
+
+An example below the approximate output for the groups endpoint for one group:
+
+```json
+[
+ {
+ "groupName" : "tester",
+ "applications": ["app2"],
+ "queues":
+ {
+ "queuename": "root",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app2"],
+ "children": [
+ {
+ "queuename": "root.test",
+ "resourceUsage": {
+ "memory": 6000000000,
+ "vcore": 6000
+ },
+ "runningApplications": ["app2"],
+ "children": []
+ }
+ ]
+ }
+ }
+]
+```
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/developer_guide/build.md b/versioned_docs/version-1.5.0/developer_guide/build.md
new file mode 100644
index 0000000..55f71e8
--- /dev/null
+++ b/versioned_docs/version-1.5.0/developer_guide/build.md
@@ -0,0 +1,244 @@
+---
+id: build
+title: Build and Run
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn always works with a container orchestrator system. Currently, a
+Kubernetes shim ([yunikorn-k8shim](https://github.com/apache/yunikorn-k8shim))
+is provided which provides a drop-in scheduler for the Kubernetes platform.
+This document describes how to setup and use a local development environment.
+
+## Dev Environment setup
+
+Read the [Dev Environment Setup](developer_guide/env_setup.md) guide first to
+setup Docker and Kubernetes development environment.
+
+## Build YuniKorn
+
+Prerequisites:
+- Golang: check the `.go_version` file in the root of the repositories for the
+version Yunikorn requires. The minimum version can change per release branch.
+Using earlier Go versions will cause compilation issues.
+
+You can build the scheduler for Kubernetes from the [yunikorn-k8shim](https://github.com/apache/yunikorn-k8shim)
+project. The build procedure will build all components into a single executable
+that can be deployed and running on Kubernetes.
+
+Start the integrated build process by pulling the `yunikorn-k8shim` repository:
+```bash
+mkdir $HOME/yunikorn/
+cd $HOME/yunikorn/
+git clone https://github.com/apache/yunikorn-k8shim.git
+```
+
+At this point you have an environment that will allow you to build an
+integrated image for the YuniKorn scheduler.
+
+### Build Docker images
+
+Building the Docker images can be triggered by following command:
+```shell script
+make image
+```
+
+This will generate images for the scheduler, scheduler plugin, and admission
+controller.
+
+The images created can be deployed directly on Kubernetes.
+Some sample deployments that can be used are found under the
+[deployments/scheduler](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/scheduler)
+directory of the `yunikorn-k8shim` repository. Alternatively, the Helm charts
+located within the [helm-charts](https://github.com/apache/yunikorn-release/tree/master/helm-charts)
+directory of the `yunikorn-release` repository may be used. These match what is used
+for release builds.
+
+The configuration of YuniKorn can be customized via a ConfigMap as explained in the
+[scheduler configuration deployment](developer_guide/deployment.md) document.
+
+The `make image` build command will first build the integrated executables and
+then create the docker images. If you want to use pre-built images based on an
+offical release, please check the [Docker Hub repo](https://hub.docker.com/r/apache/yunikorn).
+
+The default image tags are not suitable for deployments to a private
+repository as these would attempt to push to Docker Hub without proper
+credentials. You *must* update the `REGISTRY` variable in the `Makefile` to
+push to an accessible repository. When you update the image tag be aware that
+the deployment examples given will also need to be updated to reflect the same
+change.
+
+### Inspect Docker images
+
+The Docker images built from previous step have embedded some important build
+info in the image metadata. You can retrieve this information with docker
+`inspect` command:
+
+```shell script
+docker inspect apache/yunikorn:scheduler-amd64-latest
+docker inspect apache/yunikorn:scheduler-plugin-amd64-latest
+docker inspect apache/yunikorn:admission-controller-amd64-latest
+```
+
+The `amd64` tag is dependent on your host architecture (i.e. for Intel it would
+be `amd64` and for Mac M1, it would be `arm64`).
+
+This info includes git revisions (last commit SHA) for each component, to help
+you understand which version of the source code was shipped by this image. They
+are listed as docker image `labels`, such as
+
+```json
+"Labels": {
+ "BuildTimeStamp": "2019-07-16T23:08:06+0800",
+ "Version": "0.1",
+ "yunikorn-core-revision": "dca66c7e5a9e",
+ "yunikorn-k8shim-revision": "bed60f720b28",
+ "yunikorn-scheduler-interface-revision": "3df392eded1f"
+}
+```
+
+### Dependencies
+
+The dependencies in the projects are managed using
+[go modules](https://blog.golang.org/using-go-modules).
+
+If you want to modify one of the projects locally and build with your local
+dependencies you will need to change the module file. Changing dependencies
+requires using `go.mod` `replace` directives as explained in the
+[Update dependencies](#updating-dependencies) section.
+
+The YuniKorn project has four code repositories:
+ - [yunikorn-scheduler-interface](https://github.com/apache/yunikorn-scheduler-interface)
+ (protobuf interface between core and shim)
+ - [yunikorn-core](https://github.com/apache/yunikorn-core)
+ (core scheduler logic)
+ - [yunikorn-k8shim](https://github.com/apache/yunikorn-k8shim)
+ (Kubernetes-specific shim)
+ - [yunikorn-web](https://github.com/apache/yunikorn-web)
+ (YuniKorn Web UI)
+
+Each of these dependencies is a Go module and there are dependencies between
+them. During the development cycle it can be required to break the dependency
+on the committed version from github. This requires making changes in the module
+file to allow loading a local copy or a forked copy from a different repository.
+
+Additionally, there are two additional auxiliary repositories:
+ - [yunikorn-release](https://github.com/apache/yunikorn-release)
+ (release management scripts and official Helm charts)
+ - [yunikorn-site](https://github.com/apache/yunikorn-site)
+ (source of the yunikorn.apache.org web site)
+
+#### Affected repositories
+The following dependencies exist between the repositories:
+
+| Repository | Depends on |
+|------------------------------|---------------------------------------------|
+| yunikorn-core | yunikorn-scheduler-interface |
+| yunikorn-k8shim | yunikorn-scheduler-interface, yunikorn-core |
+| yunikorn-scheduler-interface | none |
+| yunikorn-web | none |
+
+The `yunikorn-web` repository has no direct go dependency on the other
+repositories. However any change to the `yunikorn-core` web services can affect
+the web interface.
+
+#### Making local changes
+
+To make sure that the local changes will not break other parts of the
+build you should run:
+- A full build `make` (build target depends on the repository)
+- A full unit test run `make test`
+
+Any test failures should be fixed before proceeding.
+
+#### Updating dependencies
+
+The simplest way is to use the `replace` directive in the module file.
+The `replace` directive allows you to override the import path with a new
+(local) path. There is no need to change any of the imports in the source code.
+The change must be made in the `go.mod` file of the repository that has the
+dependency.
+
+Using `replace` to use of a forked dependency, such as:
+```
+replace github.com/apache/yunikorn-core => example.com/some/forked-yunikorn
+```
+
+There is no requirement to fork and create a new repository. If you do not have
+a repository you can use a local checked out copy too.
+
+Using `replace` to use of a local directory as a dependency:
+```
+replace github.com/apache/yunikorn-core => /User/example/local/checked-out-yunikorn
+```
+
+For the same dependency using a relative path:
+```
+replace github.com/apache/yunikorn-core => ../checked-out-yunikorn
+```
+Note: if the `replace` directive is using a local filesystem path, then the target
+must have a `go.mod` file at that location.
+
+Further details can be found on the Go Wiki:
+[When should I use the 'replace' directive?](https://github.com/golang/go/wiki/Modules#when-should-i-use-the-replace-directive)
+
+## Build the Web UI
+
+Example deployments reference the
+[YuniKorn Web UI](https://github.com/apache/yunikorn-web). The `yunikorn-web`
+project has specific requirements for the build. Follow the steps in the
+[README](https://github.com/apache/yunikorn-web/blob/master/README.md) to prepare
+a development environment and build the Web UI. However, the scheduler is fully
+functional without the Web UI.
+
+## Run YuniKorn locally
+
+When you have a local development environment setup you can run the scheduler
+in your local Kubernetes environment. This has been tested in a desktop
+enviornment with Docker Desktop, Minikube, and Kind. See the
+[Dev Environment Setup](developer_guide/env_setup.md) guide for further details.
+
+To run a local instance of the scheduler:
+
+```shell script
+make run
+```
+
+This will launch a local scheduler and connect to the Kubernetes cluster
+referenced in your `KUBECONFIG` or `$HOME/.kube/config`.
+
+To run YuniKorn in Kubernetes scheduler plugin mode instead, execute:
+
+```
+make run_plugin
+```
+
+You can also use the same approach to run the scheduler locally but connecting
+to a remote kubernetes cluster, as long as the `$HOME/.kube/config` file
+is pointing to that remote cluster.
+
+## Run end-to-end tests
+
+In addition to the unit tests for each project, YuniKorn contains many e2e
+(end-to-end) tests in the `yunikorn-k8shim` repository which validate
+functionaliy of the scheduler on a functioning Kubernetes cluster.
+
+How to run the tests locally is described
+[here](https://github.com/apache/yunikorn-k8shim/blob/master/test/e2e/README.md).
diff --git a/versioned_docs/version-1.5.0/developer_guide/dependencies.md b/versioned_docs/version-1.5.0/developer_guide/dependencies.md
new file mode 100644
index 0000000..d458798
--- /dev/null
+++ b/versioned_docs/version-1.5.0/developer_guide/dependencies.md
@@ -0,0 +1,124 @@
+---
+id: dependencies
+title: Go module updates
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## When to update
+The references in the `master` branches must be updated if a change is made in the scheduler interface.
+Updating the dependency of a shim in reference to the core might be needed even if the scheduler interface does not change.
+New functionality could be added that rely on changed field content of the messages, not the field layout of the message.
+In that case just the shim dependency needs to be updated.
+
+## Why a pseudo version
+In the `master` branch we **must** use a pseudo version for all the YuniKorn repository references we use.
+As the branch is in active development and not released we do not have a real version tag to reference.
+However, we still need to be able to point to the right commit for the dependencies.
+
+Go allows using [pseudo versions](https://go.dev/ref/mod#pseudo-versions) for these specific cases.
+An example of the pseudo versions we use in the Kubernetes shim:
+```
+module github.com/apache/yunikorn-k8shim
+
+go 1.16
+
+require (
+ github.com/apache/yunikorn-core v0.0.0-20220325135453-73d55282f052
+ github.com/apache/yunikorn-scheduler-interface v0.0.0-20220325134135-4a644b388bc4
+ ...
+)
+```
+Release branches **must** not use pseudo versions.
+During the creation of a release, [tags](/community/release_procedure#tag-and-update-release-for-version) will be created.
+These tags will be used as the reference in the go.mod files for the release.
+
+## Enforcement of pseudo version
+In the pull request checks for the `yunikorn-core` and `yunikorn-k8shim` repositories enforce the format of the versions.
+A build failure will be triggered if the version reference for the `yunikorn-core` or `yunikorn-scheduler-interface`
+repositories in the `master` branch is not a pseudo version.
+
+The check enforces that the start of the version reference is `v.0.0.0-`
+
+Pseudo versions are not enforced in the release branches as per [why a pseudo version](#why-a-pseudo-version) explanation above.
+
+## Updating the core dependency
+Before updating the core dependency must make sure that the scheduler interface changes are finalised.
+
+1. Make the changes in the scheduler interface.
+2. Commit the changes into the master branch on GitHub and pull the latest master branch commit.
+3. [Generate a new pseudo version](#generating-a-pseudo-version) for the scheduler-interface.
+
+Updating the core dependency
+
+4. Update the go.mod file for the dependent repository: core repository
+ * Open the go.mod file
+ * Copy the generated pseudo version reference
+ * Replace the scheduler-interface version reference with the one generated in step 3.
+ * Save the go.mod file
+5. Run a `make test` to be sure that the change works. The build will pull down the new dependency and the change in the scheduler interface will be used.
+6. Commit the changes into the master branch on GitHub and pull the latest master branch commit
+
+## Updating a shim dependency
+Before updating a shim dependency you must make sure that the core dependency has been updated and committed.
+There are cases that the reference for the scheduler-interface has not changed.
+This is not an issue, either skip the update steps or execute them as per normal resulting in no changes as part of the commit.
+
+7. [Generate a new pseudo version](#generating-a-pseudo-version) for the core
+8. Update the go.mod file for the dependent repository: k8shim repository
+ * Open the go.mod file
+ * Copy the generated pseudo version reference of the scheduler interface
+ * Replace the scheduler-interface version reference with the one generated in step 3.
+ * Copy the generated pseudo version reference of the core
+ * Replace the core version reference with the one generated in step 7.
+ * Save the go.mod file
+9. Run a `make test` to be sure that the change works. The build will pull down the new dependency and the changes in the core and scheduler interface will be used.
+10. Commit the changes into the master branch on GitHub
+
+:::note
+If multiple PRs are being worked on in the scheduler interface and or core at the same time a different PR might have already applied the update.
+This will all depend on the commit order.
+It is therefor that steps 5 and 8 are performed to make sure there is no regression.
+:::
+## Generating a pseudo version
+
+A pseudo references for use in a go.mod file is based on the commit hash and timestamp.
+It is simple to generate one using the following steps:
+
+1. Change to the repository for which the new pseudo version needs to be generated.
+2. Update the local checked out code for the master branch to get the latest commits
+```
+git pull; git status
+```
+The status should show up to date with the `origin` from where it was cloned.
+3. Run the following command to get the pseudo version:
+```
+TZ=UTC git --no-pager show --quiet --abbrev=12 --date='format-local:%Y%m%d%H%M%S' --format='v0.0.0-%cd-%h'
+```
+4. This command will print a line like this:
+```
+v0.0.0-20220318052402-b3dfd0d2adaa
+```
+That is the pseudo version that can be used in the go.mod files.
+
+:::note
+The pseudo version must be based on a commit that is in the vcs system, i.e. from Github.
+Local commits or commits that are not yet merged in a PR cannot be used.
+:::
diff --git a/versioned_docs/version-1.5.0/developer_guide/deployment.md b/versioned_docs/version-1.5.0/developer_guide/deployment.md
new file mode 100644
index 0000000..2a098e2
--- /dev/null
+++ b/versioned_docs/version-1.5.0/developer_guide/deployment.md
@@ -0,0 +1,148 @@
+---
+id: deployment
+title: Deploy to Kubernetes
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The easiest way to deploy YuniKorn is to leverage our [helm charts](https://hub.helm.sh/charts/yunikorn/yunikorn),
+you can find the guide [here](get_started/get_started.md). This document describes the manual process to deploy YuniKorn
+scheduler and admission controller. It is primarily intended for developers.
+
+**Note** The primary source of deployment information is the Helm chart, which can be found at [yunikorn-release](https://github.com/apache/yunikorn-release/). Manual deployment may lead to out-of-sync configurations, see [deployments/scheduler](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/scheduler)
+
+## Build docker image
+
+Under project root of the `yunikorn-k8shim`, run the command to build an image using the map for the configuration:
+```
+make image
+```
+
+This command will build an image. The image will be tagged with a default version, image tag and your build architecture.
+
+**Note** the default build uses a hardcoded user and tag. You *must* update the `IMAGE_TAG` variable in the `Makefile` to push to an appropriate repository.
+
+**Note** the latest yunikorn images in docker hub are not updated anymore due to ASF policy. Hence, you should build both scheduler image and web image locally before deploying them.
+
+**Note** the imaging tagging includes your build architecture. For Intel, it would be `amd64` and for Mac M1, it would be `arm64`.
+
+## Setup RBAC for Scheduler
+In the example, RBAC are configured for the yuniKorn namespace.
+The first step is to create the RBAC role for the scheduler, see [yunikorn-rbac.yaml](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/scheduler/yunikorn-rbac.yaml)
+```
+kubectl create -f deployments/scheduler/yunikorn-rbac.yaml
+```
+The role is a requirement on the current versions of kubernetes.
+
+## Create/Update the ConfigMap
+
+This must be done before deploying the scheduler. It requires a correctly setup kubernetes environment.
+This kubernetes environment can be either local or remote.
+
+- download configuration file if not available on the node to add to kubernetes:
+```
+curl -o yunikorn-configs.yaml https://raw.githubusercontent.com/apache/yunikorn-k8shim/master/deployments/scheduler/yunikorn-configs.yaml
+```
+- modify the content of yunikorn-configs.yaml file as needed, and create ConfigMap in kubernetes:
+```
+kubectl create configmap yunikorn-configs --from-file=yunikorn-configs.yaml
+```
+- Or update ConfigMap in kubernetes:
+```
+kubectl create configmap yunikorn-configs --from-file=yunikorn-configs.yaml -o yaml --dry-run=client | kubectl apply -f -
+```
+- check if the ConfigMap was created/updated correctly:
+```
+kubectl describe configmaps yunikorn-configs
+```
+
+## Deploy the Scheduler
+
+The scheduler can be deployed with following command.
+```
+kubectl create -f deployments/scheduler/scheduler.yaml
+```
+
+The deployment will run 2 containers from your pre-built docker images in 1 pod,
+
+* yunikorn-scheduler-core (yunikorn scheduler core and shim for K8s)
+* yunikorn-scheduler-web (web UI)
+
+Alternatively, the scheduler can be deployed as a K8S scheduler plugin:
+```
+kubectl create -f deployments/scheduler/plugin.yaml
+```
+
+The pod is deployed as a customized scheduler, it will take the responsibility to schedule pods which explicitly specifies `schedulerName: yunikorn` in pod's spec. In addition to the `schedulerName`, you will also have to add a label `applicationId` to the pod.
+```yaml
+ metadata:
+ name: pod_example
+ labels:
+ applicationId: appID
+ spec:
+ schedulerName: yunikorn
+```
+
+Note: Admission controller abstracts the addition of `schedulerName` and `applicationId` from the user and hence, routes all traffic to YuniKorn. If you use helm chart to deploy, it will install admission controller along with the scheduler. Otherwise, proceed to the steps below to manually deploy the admission controller if running non-example workloads where `schedulerName` and `applicationId` are not present in the pod spec and metadata, respectively.
+
+
+## Setup RBAC for Admission Controller
+
+Before the admission controller is deployed, we must create its RBAC role, see [admission-controller-rbac.yaml](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/scheduler/admission-controller-rbac.yaml).
+
+```
+kubectl create -f deployments/scheduler/admission-controller-rbac.yaml
+```
+
+## Create the Secret
+
+Since the admission controller intercepts calls to the API server to validate/mutate incoming requests, we must deploy an empty secret
+used by the webhook server to store TLS certificates and keys. See [admission-controller-secrets.yaml](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/scheduler/admission-controller-secrets.yaml).
+
+```
+kubectl create -f deployments/scheduler/admission-controller-secrets.yaml
+```
+
+## Deploy the Admission Controller
+
+Now we can deploy the admission controller as a service. This will automatically validate/modify incoming requests and objects, respectively, in accordance with the [example in Deploy the Scheduler](#Deploy-the-Scheduler). See the contents of the admission controller deployment and service in [admission-controller.yaml](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/scheduler/admission-controller.yaml).
+
+```
+kubectl create -f deployments/scheduler/admission-controller.yaml
+```
+
+## Access to the web UI
+
+When the scheduler is deployed, the web UI is also deployed in a container.
+Port forwarding for the web interface on the standard ports can be turned on via:
+
+```
+kubectl port-forward svc/yunikorn-service 9889 9080
+```
+
+`9889` is the default port for Web UI, `9080` is the default port of scheduler's Restful service where web UI retrieves info from.
+Once this is done, web UI will be available at: http://localhost:9889.
+
+## Configuration Hot Refresh
+
+YuniKorn uses config maps for configurations, and it supports loading configuration changes automatically by watching config map changes using shared informers.
+
+To make configuration changes, simply update the content in the configmap, which can be done either via Kubernetes dashboard UI or command line. Note, changes made to the configmap might have some delay to be picked up by the scheduler.
+
diff --git a/versioned_docs/version-1.5.0/developer_guide/env_setup.md b/versioned_docs/version-1.5.0/developer_guide/env_setup.md
new file mode 100644
index 0000000..4fd59a6
--- /dev/null
+++ b/versioned_docs/version-1.5.0/developer_guide/env_setup.md
@@ -0,0 +1,299 @@
+---
+id: env_setup
+title: Dev Environment Setup
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+There are several ways to setup a local development environment for Kubernetes.
+The three most common ones are **Minikube** ([docs](https://kubernetes.io/docs/setup/minikube/)),
+**Docker Desktop** and **Kind** ([docs](https://kind.sigs.k8s.io/)).
+**Minikube** provisions a local Kubernetes cluster on several Virtual Machines
+(via VirtualBox or something similar).
+
+**Docker Desktop**, on the other hand, sets up Kubernetes cluster using a local
+Docker installation.
+
+**Kind** provides lightweight Kubernetes clusters for Windows, Linux and Mac
+using an existing Docker installation.
+
+## Local Kubernetes cluster using Docker Desktop
+
+In this tutorial, we will base all the installs on Docker Desktop.
+Even in this case we can use a lightweight [minikube](#local-kubernetes-cluster-with-minikube)
+setup which gives the same functionality with less impact.
+
+### Installation
+
+Download and install [Docker Desktop](https://www.docker.com/products/docker-desktop).
+Newer Docker versions have an embedded version of Kubernetes so no additional
+installation is needed. Follow the instructions [here](https://docs.docker.com/desktop/kubernetes/#install-and-turn-on-kubernetes)
+to get Kubernetes up and running within Docker Desktop.
+Alternatively, a Kind cluster may be created (see instructions
+[here](https://kind.sigs.k8s.io/docs/user/quick-start/#creating-a-cluster).
+
+Once Kubernetes is started in Docker Desktop, you should see something similar
+to this:
+
+
+
+This means that:
+
+1. Kubernetes is running.
+2. The command line tool `kubectl` is installed in the `/usr/local/bin` directory.
+3. The Kubernetes context is set to `docker-desktop`.
+
+### Deploy and access dashboard
+
+Optionally, after setting up Kubernetes you may wish to deploy the Kubernetes
+Dashboard Web UI. The dashboard may be deployed using the following steps:
+
+1. Follow the instructions [here](https://github.com/kubernetes/dashboard) to deploy the dashboard.
+2. Start the Kubernetes proxy in the background from a terminal to get access on the dashboard on the local host:
+ ```shell script
+ kubectl proxy &
+ ```
+3. Access the dashboard [here](http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/#!/login).
+
+### Access local Kubernetes cluster
+
+The dashboard as deployed in the previous step requires a token or config to
+sign in. Here we use the token to sign in. The token is generated
+automatically and can be retrieved from the system.
+
+1. Retrieve the name of the dashboard token:
+ ```shell script
+ kubectl -n kube-system get secret | grep kubernetes-dashboard-token
+ ```
+2. Retrieve the content of the token. Note that the token name ends with a random
+ 5 character code and needs to be replaced with the result of step 1. As an
+ example:
+ ```shell script
+ kubectl -n kube-system describe secret kubernetes-dashboard-token-tf6n8
+ ```
+3. Copy the token value which is part of the `Data` section with the tag `token`.
+4. Select the **Token** option in the dashboard web UI:<br/>
+ 
+5. Paste the token value into the input box and sign in:<br/>
+ 
+
+## Local Kubernetes cluster with Minikube
+Minikube can be added to an existing Docker Desktop install. Minikube can
+either use the pre-installed hypervisor or use a hypervisor of your choice.
+These instructions use [HyperKit](https://github.com/moby/hyperkit) which is
+embedded in Docker Desktop.
+
+If you want to use a different hypervisor then HyperKit make sure that you
+follow the generic minikube install instructions. Do not forget to install
+the correct driver for the chosen hypervisor if required. The minikube
+installation instructions can be found [here](https://kubernetes.io/docs/tasks/tools/install-minikube/).
+
+Docker Desktop should have already installed HyperKit. To verify this, open a
+terminal and run: `hyperkit`. Any response other than
+`hyperkit: command not found` confirms that HyperKit is installed and on
+the path. If it is not found you can choose a different hypervisor or
+fix the Docker Desktop install.
+
+### Installing Minikube
+1. Install minikube, either via `brew` or directly via these steps:
+ ```shell script
+ curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-darwin-amd64
+ chmod +x minikube
+ sudo mv minikube /usr/local/bin
+ ```
+2. Install HyperKit driver (required). You can either use `brew` or directly via these steps:
+ ```shell script
+ curl -LO https://storage.googleapis.com/minikube/releases/latest/docker-machine-driver-hyperkit
+ sudo install -o root -g wheel -m 4755 docker-machine-driver-hyperkit /usr/local/bin/
+ ```
+3. Update the minikube configuration to default to using HyperKit:
+ ```shell script
+ minikube config set vm-driver hyperkit
+ ```
+4. Change Docker Desktop to use minikube for Kubernetes:<br/>
+ 
+
+### Deploy and access the cluster
+After the installation is done you can start a new cluster.
+1. Start the minikube cluster:
+ ```shell script
+ minikube start --kubernetes-version v1.24.7
+ ```
+2. Start the minikube dashboard:
+ ```shell script
+ minikube dashboard &
+ ```
+
+### Build impact
+When you create images make sure that the build is run after pointing it to
+the correct cluster. Without setting the environment minikube might not find
+the docker images when deploying the scheduler.
+
+1. Make sure minikube is started.
+2. In the terminal where you will run the build, execute:
+ ```shell script
+ eval $(minikube docker-env)
+ ```
+3. Run the image build from the yunikorn-k8shim repository root:
+ ```shell script
+ make image
+ ```
+4. Deploy the scheduler as per the normal instructions.
+
+## Local Kubernetes Cluster with Kind
+
+Kind (Kubernetes in Docker) is a lightweight tool for running lightweight
+Kubernetes environments. It is very easy to test different Kubernetes versions
+with Kind by specifing the version during cluster setup.
+
+### Installation
+
+If you have go installed, you can run:
+```shell script
+go install sigs.k8s.io/kind@latest
+```
+
+Other installation methods can be found on the Kind
+[website](https://kind.sigs.k8s.io/docs/user/quick-start/#installation).
+
+Kind version 0.15 is required for Kubernetes 1.25 or later.
+Later versions of Kind add Kubernetes 1.26, 1.27 and 1.28.
+Check the Kind release notes for the specific Kubernetes releases supported.
+
+### Using Kind
+
+To test a new version of Kubernetes, you can pull a corresponding image from
+kind's repository.
+
+For example, to create a cluster running Kubernetes 1.26.6:
+```shell script
+kind create cluster --name test --image kindest/node:v1.26.6
+```
+
+Kind will download the appropriate image and launch a new cluster named
+`test`. The active Kubernetes cluster will also be changed to `test`.
+
+To delete the kind cluster:
+```shell script
+kind delete cluster --name test
+```
+
+### Loading your images
+
+In order to use a local image, you have to load your images into kind's
+registry. If you run `make image`, you could use the following command
+to load your kind image. This assumes AMD64 architecture.
+
+The scheduler, web-ui and admission-controller examples are below:
+
+```shell script
+kind load docker-image apache/yunikorn:scheduler-amd64-latest
+kind load docker-image apache/yunikorn:web-amd64-latest
+kind load docker-image apache/yunikorn:admission-amd64-latesta
+```
+
+If running on an ARM system, replace `amd64` with `arm64` above.
+
+## Debug code locally
+
+The scheduler may be run locally for debugging. This example assumes
+you have installed the GoLand IDE for development.
+
+In GoLand, open the `yunikorn-k8shim` project. Then click "Run" ->
+"Debug..." -> "Edit Configuration..." to get the pop-up configuration
+window. Note, you need to click "+" to create a new profile if the `Go Build`
+option is not available at the first time.
+
+
+
+Set the following values in the dialog (as shown):
+
+- Run Kind: Package
+- Package path: `github.com/apache/yunikorn-k8shim/pkg/cmd/shim`
+- Working directory: Project base directory (`yunikorn-k8shim`)
+- Program arguments: Empty
+- Environment: If `KUBECONFIG` is not set globally, ensure it is set here.
+ Additionally, you may want to set `NAMESPACE=yunikorn`, as otherwise
+ YuniKorn will look for the `yunikorn-configs` ConfigMap under the
+ `default` Kubernetes namespace.
+
+Once the changes are done, click "Apply", then "Debug". You will need to
+set proper breakpoints in order to debug the program.
+
+## Debug the scheduler plugin
+
+The scheduler may also be run in plugin mode. In this mode, the YuniKorn
+scheduler is built on top of the default scheduler and runs as a
+plugin (rather than completely standalone). Functionally, it performs the
+same tasks, but relies on the upstream Kubernetes scheduler codebase for
+common functionality.
+
+The run configuration for the scheduler in plugin mode is as follows:
+
+- Run Kind: Package
+- Package path: `github.com/apache/yunikorn-k8shim/pkg/cmd/schedulerplugin`
+- Working directory: Project base directory (`yunikorn-k8shim`)
+- Program arguments:
+ ```
+ --bind-address=0.0.0.0
+ --leader-elect=false
+ --config=conf/scheduler-config-local.yaml
+ -v=2
+ ```
+- Environment: If `KUBECONFIG` is not set globally, ensure it is set here.
+ Additionally, you may want to set `NAMESPACE=yunikorn`, as otherwise
+ YuniKorn will look for the `yunikorn-configs` ConfigMap under the
+ `default` Kubernetes namespace.
+
+Additionally, before running for the first time, run `make init` from a
+terminal in the root of the `yunikorn-k8shim` repository. This will
+generate the contents of `conf/scheduler-config-local.yaml`, which is
+required.
+
+## Access remote Kubernetes cluster
+
+This setup assumes you have already installed a remote Kubernetes cluster.
+For a generic view on how to access a multiple cluster and integrate it follow
+the [accessing multiple clusters](https://kubernetes.io/docs/tasks/access-application-cluster/configure-access-multiple-clusters/)
+documentation from Kubernetes.
+
+Or follow these simplified steps:
+1. Get the Kubernetes `config` file from remote cluster, copy it to the local
+ machine and give it a unique name i.e. `config-remote`
+2. Save the `KUBECONFIG` environment variable (if set)
+ ```shell script
+ export KUBECONFIG_SAVED=$KUBECONFIG
+ ```
+3. Add the new file to the environment variable
+ ```shell script
+ export KUBECONFIG=$KUBECONFIG:config-remote
+ ```
+4. Run the command `kubectl config view` to check that both configs can be accessed
+5. Switch context using `kubectl config use-context remote-cluster`
+6. Confirm that the current context is now switched to the remote cluster config:
+ ```text
+ kubectl config get-contexts
+ CURRENT NAME CLUSTER AUTHINFO
+ docker-desktop docker-desktop-cluster docker-for-desktop
+ * remote-cluster kubernetes kubernetes-admin
+ ```
+
+More documentation can be found
+[here](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/).
diff --git a/versioned_docs/version-1.5.0/developer_guide/openshift_development.md b/versioned_docs/version-1.5.0/developer_guide/openshift_development.md
new file mode 100644
index 0000000..dcdcd53
--- /dev/null
+++ b/versioned_docs/version-1.5.0/developer_guide/openshift_development.md
@@ -0,0 +1,189 @@
+---
+id: openshift_development
+title: Development in CodeReady Containers
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn is tested against OpenShift and developers can set up their local environment to test patches against OpenShift.
+Our recommended local environment uses CodeReady containers.
+
+## Set up a running CRC cluster
+
+1. Download CodeReady Container binaries
+
+ Select your OS from the dropdown list then click on "Download" (On a Mac, you'll download crc-macos-amd64.tar.xz; on Linux, crc-linux-amd64.tar.xz).
+ You'll be asked to connect using your Red Hat login. If you don't have one, just click on "Create one now". You do *not* need a Red Hat subscription for this.
+
+ Once logged in, download CodeReady Containers binary and the pull secret.
+
+1. Unzip the tar file.
+
+ ```bash
+ tar -xvzf crc-macos-amd64.tar.xz
+ ```
+
+1. Move the crc binary under your path. Like
+
+ ```bash
+ sudo cp `pwd`/crc-macos-$CRCVERSION-amd64/crc /usr/local/bin
+ ```
+
+1. Configure CRC in accordance with your hardware capabilities.
+
+ ```bash
+ crc config set memory 16000
+ crc config set cpus 12
+ crc setup
+ ```
+1. Start the CRC and open the console.
+
+ ```bash
+ crc start --pull-secret-file pull-secret.txt
+ crc console
+ ```
+
+## Testing a patch
+
+The following steps assume you have a running CRC cluster in your laptop. Note that these steps are not tested against a remote CRC cluster.
+
+1. Access your environment through the `oc` command.
+
+ Type in the `crc oc-env` command to a shell.
+ ```bash
+ $ crc oc-env
+ export PATH="/Users/<user>/.crc/bin/oc:$PATH"
+ # Run this command to configure your shell:
+ # eval $(crc oc-env)
+ ```
+ So you need to type in this to access the `oc` comamnd:
+ ```
+ eval $(crc oc-env)
+ ```
+
+1. Log in to `oc`. After the CRC has started it will display a similar message:
+
+ ```
+ To access the cluster, first set up your environment by following 'crc oc-env' instructions.
+ Then you can access it by running 'oc login -u developer -p developer https://api.crc.testing:6443'.
+ To login as an admin, run 'oc login -u kubeadmin -p duduw-yPT9Z-hsUpq-f3pre https://api.crc.testing:6443'.
+ To access the cluster, first set up your environment by following 'crc oc-env' instructions.
+ ```
+
+ Use the `oc login -u kubeadmin ...` command.
+
+1. Get the URL of the local OpenShift cluster's internal private Docker repository by typing the command below.
+
+ ```bash
+ $ oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}'
+ default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+ By default it should be `default-route-openshift-image-registry.apps-crc.testing`. Change the steps above, if the displayed URL is different.
+
+1. Prepare the Docker images.
+
+ You can read more about this at the bottom, in the *Using custom images* section.
+
+1. Prepare the helm chart.
+
+ If you want to use custom Docker images, replace the images in the chart's `values.yaml` config file.
+
+ Note that if you manually pushed the Docker image to the `default-route-openshift-image-registry.apps-crc.testing` docker registry directly you need to have valid certs to access it.
+ On OpenShift there's service for this: `image-registry.openshift-image-registry.svc`, which is easier to use.
+
+ For example, if you want to override all of the Docker images you should use the following configs:
+ ```yaml
+ image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn
+ tag: scheduler-latest
+ pullPolicy: Always
+
+ pluginImage:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn
+ tag: scheduler-plugin-latest
+ pullPolicy: Always
+
+ admissionController:
+ image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn
+ tag: admission-latest
+ pullPolicy: Always
+
+ web:
+ image:
+ repository: image-registry.openshift-image-registry.svc:5000/yunikorn/yunikorn-web
+ tag: latest
+ pullPolicy: Always
+ ```
+
+ You can find it in the yunikorn-release repo's helm chart directory.
+
+1. Install the helm charts.
+
+ ```bash
+ helm install yunikorn . -n yunikorn
+ ```
+
+## Using custom images
+
+### Podman
+
+1. Log in into Podman using the following command.
+
+ ```bash
+ podman login --tls-verify=false -u kubeadmin -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+1. Build the image in the repository e.g. in shim using the generic `make image` command.
+
+1. Verify that the image is present in the repository.
+
+ ```bash
+ podman images
+ REPOSITORY TAG IMAGE ID CREATED SIZE
+ localhost/apache/yunikorn admission-latest 19eb41241d64 About a minute ago 53.5 MB
+ localhost/apache/yunikorn scheduler-latest e60e09b424d9 About a minute ago 543 MB
+ ```
+
+## Directly pushing OS Image Registry
+
+1. Create the images that you wish to replace.
+
+ You can either build new images locally or use official (maybe mix both).
+ * For the -shim and -web images checkout the repository (optionally make your changes) and type the following command:
+ ```bash
+ make clean image REGISTRY=default-route-openshift-image-registry.apps-crc.testing/<project>/<name>:<tag>
+ ```
+ Note that in OpenShift a project is equivalent a Kubernetes namespace. The `yunikorn` project/namespace is recommended.
+ * Using an official image is possible by, retagging it with by the `docker tag` command.
+ ```bash
+ docker tag apache/yunikorn:scheduler-latest default-route-openshift-image-registry.apps-crc.testing/yunikorn/yunikorn:scheduler-latest
+ ```
+
+1. Login to the Docker repository.
+ ```bash
+ docker login -u kubeadmin -p $(oc whoami -t) default-route-openshift-image-registry.apps-crc.testing
+ ```
+
+1. Push the Docker images to the internal Docker repository
+ ```
+ docker push default-route-openshift-image-registry.apps-crc.testing/yunikorn/yunikorn:scheduler-latest
+ ```
diff --git a/versioned_docs/version-1.5.0/developer_guide/scheduler_object_states.md b/versioned_docs/version-1.5.0/developer_guide/scheduler_object_states.md
new file mode 100644
index 0000000..86cc135
--- /dev/null
+++ b/versioned_docs/version-1.5.0/developer_guide/scheduler_object_states.md
@@ -0,0 +1,123 @@
+---
+id: scheduler_object_states
+title: Scheduler Object States
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+The YuniKorn project uses state machines to track the states of different objects.
+This ranges from applications in the core to nodes in the k8shim.
+The state machines are independent and not shared between the resource managers and core.
+A resource manager shim, and the core can thus have an independent idea of the state of a similar object.
+
+## Core Scheduler
+State change are triggered by events that get processed.
+One event can cause a change for multiple states or no change at all.
+
+### Application State
+Applications have a complex state model.
+An application when created starts ain the new state.
+
+An application can have the following states:
+* New: A new application that is being submitted or created, from here the application transitions into the accepted state when it is ready for scheduling.
+The first ask to be added will trigger the transition.
+* Accepted: The application is ready and part of the scheduling cycle.
+On allocation of the first ask the application moves into a starting state.
+This state is part of the normal scheduling cycle.
+* Starting: The application has exactly one allocation confirmed this corresponds to one running container/pod.
+The application transitions to running if and when more allocations are added to the application.
+This state times out automatically to prevent applications that consist of just one allocation from getting stuck in this state.
+The current time out is set to 5 minutes, and cannot be changed.
+If after the timeout expires the application will auto transition to running.
+The state change on time out is independent of the number of allocations added.
+This state is part of the normal scheduling cycle.
+* Running: The state in which the application will spend most of its time.
+Containers/pods can be added to and removed from the application.
+This state is part of the normal scheduling cycle.
+* Completing: An application that has no pending requests or running containers/pod will be completing.
+This state shows that the application has not been marked completed yet but currently is not actively being scheduled.
+* Completed: An application is considered completed when it has been in the completing state for a defined time period.
+From this state the application can only move to the Expired state, and it cannot move back into any of scheduling states (Running or Completing)
+The current timeout is set to 30 seconds.
+* Expired: The completed application is tracked for a period of time, after that is expired and deleted from the scheduler.
+This is a final state and after this state the application cannot be tracked anymore.
+* Failing: An application marked for failing, what still has some allocations or asks what needs to be cleaned up before entering into the Failed state.
+ The application can be Failing when the partition it belongs to is removed or during gang scheduling, if the placeholder processing times out, and the application has no real allocations yet.
+* Failed: An application is considered failed when it was marked for failure and all the pending requests and allocations were already removed.
+From this state the application can only move to the Expired state.
+* Rejected: The application was rejected when it was added to the scheduler.
+This only happens when a resource manager tries to add a new application, when it gets created in a New state, and the scheduler rejects the creation.
+Applications can be rejected due ACLs denying access to a queue the application has specified, or a placement via placement rules has failed. From this state the application can only move to the Expired state.
+
+The events that can trigger a state change:
+* Reject: rejecting the application by the scheduler (source: core scheduler)
+* Run: progress an application to the next active state (source: core scheduler)
+* Complete: mark an application as idle or complete (source: core scheduler)
+* Fail: fail an application (source: resource manager or core scheduler)
+* Expire: progress the application to the expired state and remove it from the scheduler (source: core scheduler)
+
+Here is a diagram that shows the states with the event that causes the state to change:
+
+
+### Object State
+<!-- fix the draining to stopped transition -->
+The object state is used by the following objects:
+* queues
+* partitions
+
+The object states are as follows:
+* Active: The object is active and used during the scheduling cycle.
+This is the starting and normal state of an object.
+An active object transitions to draining when it is removed.
+* Stopped: The object is stopped and no longer actively scheduled.
+The object if empty is ready to be removed from the scheduler.
+The object can transition back into active state if it gets re-started.
+* Draining: Before an object can be removed it needs to be cleaned up.
+The cleanup starts with placing the object in the draining state.
+In this state it does not accept additions or changes but is still actively being scheduled.
+This allows for a graceful shutdown, cleanup and removal of the object.
+This is the final state.
+
+The events that can trigger a state change:
+* Start: make the object active (source: core scheduler)
+* Stop: make the object inactive (source: core scheduler)
+* Remove: mark an object for removal (source: core scheduler)
+
+Here is a diagram that shows the states with the event that causes the state to change:
+
+
+### Node
+<!-- should start using object state -->
+Node objects in the core are not using a state machine but do have a state.
+A node can have one of two states: `schedulable` or `not schedulable`.
+There is no complex state model or complex transition logic.
+The scheduler can either use the node or not.
+
+The node status changes based on the status provided by the resource manager (shim) that owns the node.
+
+## K8Shim Resource Manager
+
+### Application
+
+
+### Task
+
+
+### Node
+
diff --git a/versioned_docs/version-1.5.0/get_started/core_features.md b/versioned_docs/version-1.5.0/get_started/core_features.md
new file mode 100644
index 0000000..ca9f701
--- /dev/null
+++ b/versioned_docs/version-1.5.0/get_started/core_features.md
@@ -0,0 +1,95 @@
+---
+id: core_features
+title: Features
+keywords:
+ - feature
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The main features of YuniKorn include:
+
+## App-aware scheduling
+One of the key differences of YuniKorn is that it does app-aware scheduling. The default K8s scheduler simply schedules
+pod by pod without any context about user, app, or queue. However, YuniKorn recognizes users, apps, and queues, and it considers
+a lot more factors, e.g resource, ordering etc, while making scheduling decisions. This gives us the possibility to use
+fine-grained controls on resource quotas, resource fairness, and priorities, which are the most important requirements
+for a multi-tenancy computing system.
+
+## Hierarchy Resource Queues
+Hierarchy queues provide an efficient mechanism to manage cluster resources. The hierarchy of the queues can logically
+map to the structure of an organization. This gives fine-grained control over resources for different tenants. The YuniKorn
+UI provides a centralised view to monitor the usage of resource queues and helps you to gain insight into how the resources are
+used across different tenants. What's more, by leveraging the min/max queue capacity, it can define how elastic it can be
+in terms of the resource consumption for each tenant.
+
+## Gang Scheduling
+An application can request a set of resources, i.e. a gang, to be scheduled all at once.
+The gang defines all the resources the application requires to start.
+During the first scheduling phase all resources requested will be reserved.
+The application will only be started when all requested resources are available.
+
+Reservation duration and application behaviour when the reservation fails are configurable.
+It is even possible to create multiple gangs of different specifications for one application.
+See the [gang design](design/gang_scheduling.md) and the Gang Scheduling [user guide](user_guide/gang_scheduling.md) for more details.
+
+## Job Ordering and Queuing
+Applications can be properly queued in working-queues, the ordering policy determining which application can get resources first.
+There are various policies such as simple `FIFO`, `Fair`, `StateAware`, or `Priority` based. Queues can maintain the order of applications,
+and based on different policies, the scheduler allocates resources to jobs accordingly. The behavior is much more predictable.
+
+What's more, when the queue max-capacity is configured, jobs and tasks can be properly queued up in the resource queue.
+If the remaining capacity is not enough, they can be waiting in line until some resources are released. This simplifies
+the client side operation. Unlike the default scheduler, resources are capped by namespace resource quotas which
+are enforced by the quota-admission-controller. If the underlying namespace does not have enough quota, pods cannot be
+created. Client side needs complex logic, e.g retry by condition, to handle such scenarios.
+
+## Resource fairness
+In a multi-tenant environment, a lot of users share cluster resources. To prevent tenants from competing for resources
+and potentially getting starved, more fine-grained fairness controls are needed to achieve fairness across users, as well as across teams/organizations.
+With consideration of weights or priorities, more important applications can demand resources beyond their share.
+This feature is often considered in relation to resource budgets, where a more fine-grained fairness mode can further improve spending efficiency.
+
+## Resource Reservation
+YuniKorn automatically does reservations for outstanding requests. If a pod could not be allocated, YuniKorn will try to
+reserve it on a qualified node and tentatively allocate the pod on this reserved node (before trying rest of nodes).
+This mechanism can prevent the pod from being starved by future smaller, less-picky pods.
+This feature is important in the batch workloads scenario because when a large amount of heterogeneous pods are submitted
+to the cluster, it's very likely some pods can be starved even when they are submitted much earlier.
+
+## Preemption
+YuniKorn's preemption feature allows higher-priority tasks to dynamically reallocate resources by preempting lower-priority ones, ensuring critical workloads get necessary resources in a multi-tenant Kubernetes environment.
+This proactive mechanism maintains system stability and fairness, integrating with Kubernetes' priority classes and YuniKorn's hierarchical queue system.
+
+## Throughput
+Throughput is a key criterion for measuring scheduler performance. It is critical for a large scale distributed system.
+If throughput is bad, applications may waste time on waiting for scheduling and further impact service SLAs.
+When the cluster gets bigger, it also means there is a requirement for higher throughput. The [performance evaluation based on Kube-mark](performance/evaluate_perf_function_with_kubemark.md)
+reveals some perf numbers.
+
+## MaxApplication Enforcement
+The MaxApplication enforcement feature allows users to limit the number of running applications for a configured queue.
+This feature is critical in large scale batch workloads.
+Without this feature, when a large number of concurrent jobs are launched, they would compete for resources, and a certain amount of resources would be wasted, which could lead to job failure.
+The [Partition and Queue Configuration](user_guide/queue_config.md) provides configuration examples.
+
+## CPU Architecture support
+YuniKorn supports running on ARM as well as on AMD/Intel CPUs.
+With the release of YuniKorn 1.1.0, prebuilt convenience images for both architectures are provided in docker hub.
diff --git a/versioned_docs/version-1.5.0/get_started/get_started.md b/versioned_docs/version-1.5.0/get_started/get_started.md
new file mode 100644
index 0000000..2accf66
--- /dev/null
+++ b/versioned_docs/version-1.5.0/get_started/get_started.md
@@ -0,0 +1,83 @@
+---
+id: user_guide
+title: Get Started
+slug: /
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Before reading this guide, we assume you either have a Kubernetes cluster, or a local Kubernetes dev environment, e.g MiniKube.
+It is also assumed that `kubectl` is on your path and properly configured.
+Follow this [guide](developer_guide/env_setup.md) on how to setup a local Kubernetes cluster using docker-desktop.
+
+## Install
+
+The easiest way to get started is to use our Helm Charts to deploy YuniKorn on an existing Kubernetes cluster.
+It is recommended to use Helm 3 or later versions.
+
+```shell script
+helm repo add yunikorn https://apache.github.io/yunikorn-release
+helm repo update
+kubectl create namespace yunikorn
+helm install yunikorn yunikorn/yunikorn --namespace yunikorn
+```
+
+By default, the helm chart will install the scheduler, web-server and the admission-controller in the cluster.
+When `admission-controller` is installed, it simply routes all traffic to YuniKorn. That means the resource scheduling
+is delegated to YuniKorn. You can disable it by setting `embedAdmissionController` flag to `false` during the helm install.
+
+The YuniKorn scheduler can also be deployed as a Kubernetes scheduler plugin by setting the Helm `enableSchedulerPlugin`
+flag to `true`. This will deploy an alternate Docker image which contains YuniKorn compiled together with the default
+scheduler. This new mode offers better compatibility with the default Kubernetes scheduler and is suitable for use with the
+admission controller delegating all scheduling to YuniKorn. Because this mode is still very new, it is not enabled by default.
+
+If you are unsure which deployment mode you should use, refer to our [side-by-side comparison](user_guide/deployment_modes).
+
+Further configuration options for installing YuniKorn via Helm are available in the [YuniKorn Helm hub page](https://hub.helm.sh/charts/yunikorn/yunikorn).
+
+If you don't want to use helm charts, you can find our step-by-step
+tutorial [here](developer_guide/deployment.md).
+
+## Uninstall
+
+Run the following command to uninstall YuniKorn:
+```shell script
+helm uninstall yunikorn --namespace yunikorn
+```
+
+## Access the Web UI
+
+When the scheduler is deployed, the web UI is also deployed in a container.
+Port forwarding for the web interface on the standard port can be turned on via:
+
+```
+kubectl port-forward svc/yunikorn-service 9889:9889 -n yunikorn
+```
+
+`9889` is the default port for web UI.
+Once this is done, web UI will be available at: `http://localhost:9889`.
+
+
+
+YuniKorn UI provides a centralised view for cluster resource capacity, utilization, and all application info.
+
+Besides, YuniKorn also exposes its scheduling metrics via Prometheus.
+
+If you want to monitor the yunikorn core services by using Prometheus and Grafana, you can find our step-by-step tutorial [here](../user_guide/prometheus.md).
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/get_started/version.md b/versioned_docs/version-1.5.0/get_started/version.md
new file mode 100644
index 0000000..9ec67bb
--- /dev/null
+++ b/versioned_docs/version-1.5.0/get_started/version.md
@@ -0,0 +1,58 @@
+---
+id: version
+title: Version details
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Which version should be used?
+
+The recommendation is to always run the latest possible version of YuniKorn.
+New releases are made regular driven by new features for YuniKorn itself and or the shim.
+Features and fixes are currently only added to the development branch from which releases are cut.
+
+At this point in time the project only creates minor releases, no patch releases.
+
+If a shim version, i.e. Kubernetes, is supported by multiple YuniKorn releases we recommend to always use the latest YuniKorn release possible.
+Older build dependencies in earlier releases could prevent you from using some new functionality exposed by a newer shim.
+
+# Kubernetes versions supported by YuniKorn
+
+| K8s Version | Supported <br/>from version | Support ended |
+|---------------------|:---------------------------:|:-------------:|
+| 1.12.x (or earlier) | - | - |
+| 1.13.x | 0.8.0 | 0.10.0 |
+| 1.14.x | 0.8.0 | 0.10.0 |
+| 1.15.x | 0.8.0 | 0.10.0 |
+| 1.16.x | 0.10.0 | 0.11.0 |
+| 1.17.x | 0.10.0 | 0.11.0 |
+| 1.18.x | 0.10.0 | 0.11.0 |
+| 1.19.x | 0.11.0 | 1.0.0 |
+| 1.20.x | 0.12.1 | 1.2.0 |
+| 1.21.x | 0.12.1 | 1.3.0 |
+| 1.22.x | 0.12.2 | 1.3.0 |
+| 1.23.x | 0.12.2 | 1.3.0 |
+| 1.24.x | 1.0.0 | - |
+| 1.25.x | 1.2.0 | - |
+| 1.26.x | 1.2.0 | - |
+| 1.27.x | 1.4.0 | - |
+| 1.28.x | 1.4.0 | - |
+| 1.29.x | 1.5.0 | - |
+
diff --git a/versioned_docs/version-1.5.0/metrics/queue.md b/versioned_docs/version-1.5.0/metrics/queue.md
new file mode 100644
index 0000000..4943fd3
--- /dev/null
+++ b/versioned_docs/version-1.5.0/metrics/queue.md
@@ -0,0 +1,110 @@
+---
+id: queue
+title: Queue
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Application
+Each queue has a `<queue_name> queue_app` metric to trace the applications in the queue.
+`<queue_name> queue_app` metrics records the number of applications in different states.
+These application states include `running`, `accepted`, `rejected`, `failed` and `completed`.
+`<queue_name> queue_app` metrics record container states including `released`, `allocated`.
+
+Prior to `1.5.0` queue metrics were pushed to a separate subsystem for each queue. In `1.5.0`
+fixed metrics with `<queue name>` labels were introduced. Metrics using `<queue name>` subsystems
+will be **deprecated in `1.6.0` and removed in `1.7.0`** in favor of a `<queue name>` label based approach.
+
+### Label
+(Introduced in `1.5.0`)
+
+**Metric Type**: `gauge`
+
+**Namespace**: `yunikorn`
+
+**Label**: `queue: <queue name>`
+
+**TYPE**: `yunikorn_queue_app`
+
+```json
+yunikorn_queue_app{queue="root.default",state="accepted"} 3
+yunikorn_queue_app{queue="root.default",state="running"} 3
+```
+
+### Subsystem
+|:exclamation: To be deprecated in `1.6.0` and removed in `1.7.0`|
+|----------------------------------------------------------------|
+
+**Metric Type**: `gauge`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `<queue name>`
+
+**TYPE**: `yunikorn_<queue name>_queue_app`
+
+```json
+yunikorn_root_default_queue_app{state="accepted"} 3
+yunikorn_root_default_queue_app{state="running"} 3
+```
+
+## Resource
+The `<queue_name> queue_resource` metric to trace the resource in the queue.
+These resource states include `guaranteed`, `max`, `allocated`, `pending`, `preempting`.
+### Label
+(Introduced in `1.5.0`)
+
+**Metric Type**: `gauge`
+
+**Namespace**: `yunikorn`
+
+**Label**: `queue: <queue name>`
+
+**TYPE**: `yunikorn_queue_resource`
+
+```json
+yunikorn_queue_resource{queue="root",resource="ephemeral-storage",state="max"} 9.41009558e+10
+yunikorn_queue_resource{queue="root",resource="hugepages-1Gi",state="max"} 0
+yunikorn_queue_resource{queue="root",resource="hugepages-2Mi",state="max"} 0
+yunikorn_queue_resource{queue="root",resource="memory",state="max"} 1.6223076352e+10
+yunikorn_queue_resource{queue="root",resource="pods",state="max"} 110
+yunikorn_queue_resource{queue="root",resource="vcore",state="max"} 8000
+```
+
+### Subsystem
+|:exclamation: To be deprecated in `1.6.0` and removed in `1.7.0`|
+|----------------------------------------------------------------|
+
+**Metric Type**: `gauge`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `<queue name>`
+
+**TYPE**: `yunikorn_<queue name>_queue_resource`
+
+```json
+yunikorn_root_queue_resource{resource="ephemeral-storage",state="max"} 9.41009558e+10
+yunikorn_root_queue_resource{resource="hugepages-1Gi",state="max"} 0
+yunikorn_root_queue_resource{resource="hugepages-2Mi",state="max"} 0
+yunikorn_root_queue_resource{resource="memory",state="max"} 1.6223076352e+10
+yunikorn_root_queue_resource{resource="pods",state="max"} 110
+yunikorn_root_queue_resource{resource="vcore",state="max"} 8000
+```
diff --git a/versioned_docs/version-1.5.0/metrics/runtime.md b/versioned_docs/version-1.5.0/metrics/runtime.md
new file mode 100644
index 0000000..0f787df
--- /dev/null
+++ b/versioned_docs/version-1.5.0/metrics/runtime.md
@@ -0,0 +1,39 @@
+---
+id: runtime
+title: Runtime
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## MemStats
+| Yunikorn Metric | Runtime MemStats | Metric Type |
+|-------------------------|---------------------------------------------------|-----------------|
+| go_mem_stats | `Alloc`,`TotalAlloc`, `Sys`, `HeapIdle` and so on | `guage` |
+| go_pause_ns | `PauseNs` | `guage` |
+| go_pause_end | `PauseEnd` | `guage` |
+| go_alloc_bysize_maxsize | `BySize.Size` | `histogram` |
+| go_alloc_bysize_free | `BySize.Frees` | `histogram` |
+| go_alloc_bysize_malloc | `BySize.Mallocs` | `histogram` |
+
+## Generic
+The `go_generic` metric includes descriptions of supported metrics
+in the `runtime/metrics` package.
+
+**Metric Type**: `guage`
diff --git a/versioned_docs/version-1.5.0/metrics/scheduler.mdx b/versioned_docs/version-1.5.0/metrics/scheduler.mdx
new file mode 100644
index 0000000..cc186e2
--- /dev/null
+++ b/versioned_docs/version-1.5.0/metrics/scheduler.mdx
@@ -0,0 +1,314 @@
+---
+id: scheduler
+title: Scheduler
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+```mdx-code-block
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+```
+
+## Container
+### Total allocation attempt
+Total number of attempts to allocate containers.
+State of the attempt includes `allocated`, `rejected`, `error`, `released`.
+
+**Metric Type**: `counter`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+
+```json
+yunikorn_scheduler_container_allocation_attempt_total{state="allocated"} 0
+yunikorn_scheduler_container_allocation_attempt_total{state="error"} 0
+yunikorn_scheduler_container_allocation_attempt_total{state="released"} 0
+```
+
+## Application
+### Total
+Total number of applications.
+State of the application includes `running`, `failed` and `completed`.
+
+**Metric Type**: `gauge`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+```
+yunikorn_scheduler_application_total{state="running"} 0
+```
+
+### Total Submission
+Total number of application submissions.
+State of the attempt includes `accepted` and `rejected`.
+
+**Metric Type**: `counter`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+```
+yunikorn_scheduler_application_submission_total{result="accepted"} 6
+```
+
+## Latency
+### Scheduling latency
+Latency of the main scheduling routine, in milliseconds.
+This metric includes latencies, such as `Node sorting`, `Trynode` and `Trypreemption`.
+
+**Metric Type**: `histogram`
+
+**Interval**: `millisecond`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+
+```json
+yunikorn_scheduler_scheduling_latency_milliseconds_bucket{le="0.0001"} 0
+yunikorn_scheduler_scheduling_latency_milliseconds_bucket{le="0.001"} 0
+yunikorn_scheduler_scheduling_latency_milliseconds_bucket{le="0.01"} 0
+yunikorn_scheduler_scheduling_latency_milliseconds_bucket{le="0.1"} 0
+yunikorn_scheduler_scheduling_latency_milliseconds_bucket{le="1"} 0
+yunikorn_scheduler_scheduling_latency_milliseconds_bucket{le="10"} 0
+yunikorn_scheduler_scheduling_latency_milliseconds_bucket{le="+Inf"} 0
+yunikorn_scheduler_scheduling_latency_milliseconds_sum 0
+yunikorn_scheduler_scheduling_latency_milliseconds_count 0
+```
+
+### Node sorting
+Latencies including `node sorting`, `application sorting` and `queue sorting`, in milliseconds.
+
+**Metric Type**: `histogram`
+
+**Interval**: `millisecond`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+
+```mdx-code-block
+<Tabs
+ defaultValue="node_sorting"
+ values={[
+ { label: 'Node sorting', value: 'node_sorting'},
+ { label: 'App sorting', value: 'app_sorting'},
+ { label: 'Queue sorting', value: 'queue_sorting'},
+ ]}>
+<TabItem value="app_sorting">
+
+ ```json
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="app",le="0.0001"} 5
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="app",le="0.001"} 6
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="app",le="0.01"} 6
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="app",le="0.1"} 6
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="app",le="1"} 6
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="app",le="10"} 6
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="app",le="+Inf"} 6
+ yunikorn_scheduler_node_sorting_latency_milliseconds_sum{level="app"} 0.00026345400000000004
+ yunikorn_scheduler_node_sorting_latency_milliseconds_count{level="app"} 6
+ ```
+
+</TabItem>
+<TabItem value="node_sorting">
+
+ ```json
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="node",le="0.0001"} 3
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="node",le="0.001"} 3
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="node",le="0.01"} 3
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="node",le="0.1"} 3
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="node",le="1"} 3
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="node",le="10"} 3
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="node",le="+Inf"} 3
+ yunikorn_scheduler_node_sorting_latency_milliseconds_sum{level="node"} 2.5013999999999998e-05
+ yunikorn_scheduler_node_sorting_latency_milliseconds_count{level="node"} 3
+ ```
+
+</TabItem>
+<TabItem value="queue_sorting">
+
+ ```json
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="queue",le="0.0001"} 9
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="queue",le="0.001"} 9
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="queue",le="0.01"} 9
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="queue",le="0.1"} 9
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="queue",le="1"} 9
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="queue",le="10"} 9
+ yunikorn_scheduler_node_sorting_latency_milliseconds_bucket{level="queue",le="+Inf"} 9
+ yunikorn_scheduler_node_sorting_latency_milliseconds_sum{level="queue"} 4.0093e-05
+ yunikorn_scheduler_node_sorting_latency_milliseconds_count{level="queue"} 9
+ ```
+
+</TabItem>
+</Tabs>
+
+### Trynode
+Latency of node condition checks for container allocations, such as placement constraints, in milliseconds.
+
+**Metric Type**: `histogram`
+
+**Interval**: `millisecond`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+
+```json
+yunikorn_scheduler_trynode_latency_milliseconds_bucket{le="0.0001"} 0
+yunikorn_scheduler_trynode_latency_milliseconds_bucket{le="0.001"} 0
+yunikorn_scheduler_trynode_latency_milliseconds_bucket{le="0.01"} 0
+yunikorn_scheduler_trynode_latency_milliseconds_bucket{le="0.1"} 0
+yunikorn_scheduler_trynode_latency_milliseconds_bucket{le="1"} 0
+yunikorn_scheduler_trynode_latency_milliseconds_bucket{le="10"} 0
+yunikorn_scheduler_trynode_latency_milliseconds_bucket{le="+Inf"} 0
+yunikorn_scheduler_trynode_latency_milliseconds_sum 0
+yunikorn_scheduler_trynode_latency_milliseconds_count 0
+```
+
+### Trypreemption
+Latency of preemption condition checks for container allocations, in milliseconds
+
+**Metric Type**: `histogram`
+
+**Interval**: `millisecond`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+
+```json
+yunikorn_scheduler_trypreemption_latency_milliseconds_bucket{le="0.0001"} 0
+yunikorn_scheduler_trypreemption_latency_milliseconds_bucket{le="0.001"} 0
+yunikorn_scheduler_trypreemption_latency_milliseconds_bucket{le="0.01"} 0
+yunikorn_scheduler_trypreemption_latency_milliseconds_bucket{le="0.1"} 0
+yunikorn_scheduler_trypreemption_latency_milliseconds_bucket{le="1"} 0
+yunikorn_scheduler_trypreemption_latency_milliseconds_bucket{le="10"} 0
+yunikorn_scheduler_trypreemption_latency_milliseconds_bucket{le="+Inf"} 0
+yunikorn_scheduler_trypreemption_latency_milliseconds_sum 0
+yunikorn_scheduler_trypreemption_latency_milliseconds_count 0
+```
+## Node
+### Node
+Total number of nodes.
+State of the node includes `active` and `failed`.
+
+**Metric Type**: `gauge`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+
+```json
+yunikorn_scheduler_node{state="active"} 1
+yunikorn_scheduler_node{state="failed"} 0
+```
+
+### Total node usage
+`yunikorn_scheduler_<resource type>_node_usage_total`
+Total resource usage of node, by resource name.
+
+**Metric Type**: `gauge`
+
+**Namespace**: `yunikorn`
+
+**Subsystem**: `scheduler`
+
+```mdx-code-block
+<Tabs
+ defaultValue="ephemeral_storage"
+ values={[
+ { label: 'Ephemeral_storage', value: 'ephemeral_storage'},
+ { label: 'Memory', value: 'memory'},
+ { label: 'Pods', value: 'pods'},
+ { label: 'vcore', value: 'vcore'},
+ ]}>
+<TabItem value="ephemeral_storage">
+
+ ```json
+ yunikorn_scheduler_ephemeral_storage_node_usage_total
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(10%, 20%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(20%,30%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(30%,40%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(40%,50%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(50%,60%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(60%,70%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(70%,80%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(80%,90%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="(90%,100%]"} 0
+ yunikorn_scheduler_ephemeral_storage_node_usage_total{range="[0,10%]"} 1
+ ```
+
+</TabItem>
+<TabItem value="memory">
+
+ ```json
+ yunikorn_scheduler_memory_node_usage_total
+ yunikorn_scheduler_memory_node_usage_total{range="(10%, 20%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="(20%,30%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="(30%,40%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="(40%,50%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="(50%,60%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="(60%,70%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="(70%,80%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="(80%,90%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="(90%,100%]"} 0
+ yunikorn_scheduler_memory_node_usage_total{range="[0,10%]"} 1
+ ```
+
+</TabItem>
+<TabItem value="pods">
+
+ ```json
+ yunikorn_scheduler_pods_node_usage_total
+ yunikorn_scheduler_pods_node_usage_total{range="(10%, 20%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="(20%,30%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="(30%,40%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="(40%,50%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="(50%,60%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="(60%,70%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="(70%,80%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="(80%,90%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="(90%,100%]"} 0
+ yunikorn_scheduler_pods_node_usage_total{range="[0,10%]"} 1
+ ```
+
+</TabItem>
+<TabItem value="vcore">
+
+ ```json
+ yunikorn_scheduler_vcore_node_usage_total
+ yunikorn_scheduler_vcore_node_usage_total{range="(10%, 20%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="(20%,30%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="(30%,40%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="(40%,50%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="(50%,60%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="(60%,70%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="(70%,80%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="(80%,90%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="(90%,100%]"} 0
+ yunikorn_scheduler_vcore_node_usage_total{range="[0,10%]"} 1
+ ```
+
+</TabItem>
+</Tabs>
+```
diff --git a/versioned_docs/version-1.5.0/performance/evaluate_perf_function_with_kubemark.md b/versioned_docs/version-1.5.0/performance/evaluate_perf_function_with_kubemark.md
new file mode 100644
index 0000000..df244c2
--- /dev/null
+++ b/versioned_docs/version-1.5.0/performance/evaluate_perf_function_with_kubemark.md
@@ -0,0 +1,120 @@
+---
+id: evaluate_perf_function_with_kubemark
+title: Evaluate YuniKorn Performance with Kubemark
+keywords:
+ - performance
+ - throughput
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The YuniKorn community concerns about the scheduler’s performance and continues to optimize it over the releases. The community has developed some tools in order to test and tune the performance repeatedly.
+
+## Environment setup
+
+We leverage [Kubemark](https://github.com/kubernetes/kubernetes/blob/release-1.3/docs/devel/kubemark-guide.md#starting-a-kubemark-cluster) to evaluate scheduler’s performance. Kubemark is a testing tool that simulates large scale clusters. It create hollow nodes which runs hollow kubelet to pretend original kubelet behavior. Scheduled pods on these hollow nodes won’t actually execute. It is able to create a big cluster that meets our experiment requirement that reveals the yunikorn scheduler performance. Please see the [detail steps](performance/performance_tutorial.md) about how to setup the environment.
+
+## Scheduler Throughput
+
+We have designed some simple benchmarking scenarios on a simulated large scale environment in order to evaluate the scheduler performance. Our tools measure the [throughput](https://en.wikipedia.org/wiki/Throughput) and use these key metrics to evaluate the performance. In a nutshull, scheduler throughput is the rate of processing pods from discovering them on the cluster to allocating them to nodes.
+
+In this experiment, we setup a simulated 2000/4000 nodes cluster with [Kubemark](https://github.com/kubernetes/kubernetes/blob/release-1.3/docs/devel/kubemark-guide.md#starting-a-kubemark-cluster). Then we launch 10 [deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/), with setting replicas to 5000 in each deployment respectively. This simulates large scale workloads submitting to the K8s cluster simultaneously. Our tool periodically monitors and checks pods status, counting the number of started pods based on `podSpec.StartTime` as time elapses. As a comparison, we apply the same experiment to the default scheduler in the same environment. And we see the YuniKorn performance advantage over the default scheduler as illustrated below:
+
+
+<p align="center">Fig 1. Yunikorn and default scheduler throughput </p>
+
+The charts record the time spent until all pods are running on the cluster:
+
+| Number of Nodes | yunikorn | k8s default scheduler | Diff |
+|------------------ |:--------------: |:---------------------: |:-----: |
+| 2000(nodes) | 204(pods/sec) | 49(pods/sec) | 416% |
+| 4000(nodes) | 115(pods/sec) | 48(pods/sec) | 240% |
+
+In order to normalize the result, we have been running the tests for several rounds. As shown above, YuniKorn achieves a `2x` ~ `4x` performance gain comparing to the default scheduler.
+
+:::note
+
+Like other performance testing, the result is highly variable depending on the underlying hardware, e.g server CPU/memory, network bandwidth, I/O speed, etc. To get an accurate result that applies to your environment, we encourage you to run these tests on a cluster that is close to your production environment.
+
+:::
+
+## Performance Analysis
+
+The results we got from the experiment are promising. We further take a deep dive to analyze the performance by observing more internal YuniKorn metrics, and we are able to locate a few key areas affecting the performance.
+
+### K8s Limitation
+
+We found the overall performance actually is capped by the K8s master services, such as api-server, controller-manager and etcd, it did not reached the limit of YuniKorn in all our experiments. If you look at the internal scheduling metrics, you can see:
+
+
+<p align="center">Fig 2. Yunikorn metric in 4k nodes </p>
+
+Figure 2 is a screenshot from Prometheus, which records the [internal metrics](performance/metrics.md) `containerAllocation` in YuniKorn. They are the number of pods being allocated by the scheduler, but have not necessarily been bound to nodes. It consumes roughly 122 seconds to finish scheduling 50k pods, i.e 410 pod/sec. The actual throughput drops to 115 pods/sec, and the extra time was used to bind the pods on different nodes. If K8s side could catch up, we will see a better result. Actually, when we tune the performance on a large scale cluster, the first thing we do is to tune up some parameters in API-server, controller manager in order to increase the throughput. See more in the [performance tutorial doc](performance/performance_tutorial.md).
+
+### Node Sorting
+
+When the cluster size grows, we see an obvious performance drop in YuniKorn. This is because in YuniKorn, we do a full sorting of the cluster nodes in order to find the **"best-fit"** node for a given pod. Such strategy makes the pods distribution more optimal based on the [node sorting policy](./../user_guide/sorting_policies#node-sorting) being used. However, sorting nodes is expensive, doing this in the scheduling cycle creates a lot of overhead. To overcome this, we have improved our node sorting mechanism in [YUNIKORN-807](https://issues.apache.org/jira/browse/YUNIKORN-807), the idea behind of this is to use a [B-Tree](https://en.wikipedia.org/wiki/B-tree) to store all nodes and applies an incremental updates when necessary. This significantly improves the latency, according to our benchmark testing, this improves 35x, 42x, 51x, 74x respectively on 500, 1000, 2000 and 5000 nodes clusters.
+
+### Per Node Precondition Checks
+
+In each scheduling cycle, another time consuming part is the "Precondition Checks" for a node. In this phase, YuniKorn evaluates all K8s standard predicates, e.g node-selector, pod affinity/anti-affinity, etc, in order to qualify a pod is fit onto a node. These evaluations are expensive.
+
+We have done two experiments to compare the case where the predicates evaluation was enabled with being disabled. See the results below:
+
+
+<p align="center">Fig 3. Predicate effect comparison in yunikorn </p>
+
+When the predicates evaluation is disabled, the throughput improves a lot. We looked further into the latency distribution of the entire scheduling cycle and the predicates-eval latency. And found:
+
+
+<p align="center">Fig 4. predicate latency </p>
+
+
+<p align="center">Fig 5. Scheduling time with predicate active </p>
+
+
+<p align="center">Fig 6. Scheduling time with predicate inactive </p>
+
+Overall, YuniKorn scheduling cycle runs really fast, and the latency drops in **0.001s - 0.01s** range per cycle. And the majority of the time was used for predicates evaluation, 10x to other parts in the scheduling cycle.
+
+| | scheduling latency distribution(second) | predicates-eval latency distribution(second) |
+|----------------------- |:---------------------: |:---------------------: |
+| predicates enabled | 0.01 - 0.1 | 0.01-0.1 |
+| predicates disabled | 0.001 - 0.01 | none |
+
+## Why YuniKorn is faster?
+
+The default scheduler was created as a service-oriented scheduler; it is less sensitive in terms of throughput compared to YuniKorn. YuniKorn community works really hard to keep the performance outstanding in the line and keep improving it. The reasons that YuniKorn can run faster than the default scheduler are:
+
+* Short Circuit Scheduling Cycle
+
+YuniKorn keeps the scheduling cycle short and efficient. YuniKorn uses all async communication protocol to make sure all the critical paths are non-blocking calls. Most of the places are just doing in-memory calculation which can be highly efficient. The default scheduler leverages [scheduling framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/), it provides lots of flexibility to extend the scheduler, however, the trade-off is the performance. The scheduling cycle becomes to be a really long chain because it needs to visit all these plugins.
+
+* Async Events Handling
+
+YuniKorn leverages an async event handling framework to deal with internal states. And this allows the core scheduling cycle can run fast without being blocked by any expensive calls. An example is the default scheduler needs to write state updates, events to pod objects, that is done inside of the scheduling cycle. This involves persisting data to etcd which could be slow. YuniKorn, instead, caches all such events in a queue and writes back to the pod in asynchronous manner.
+
+* Faster Node Sorting
+
+After [YUNIKORN-807](https://issues.apache.org/jira/browse/YUNIKORN-807), YuniKorn does the incremental node sorting which is highly efficient. This is built on top of the so-called "resource-weight" based node scoring mechanism, and it is also extensible via plugins. All these together reduce the overhead while computing node scores. In comparison, the default scheduler provides a few extension points for calculating node scores, such as `PreScore`, `Score` and `NormalizeScore`. These computations are heavy and they are called in every scheduling cycle. See details in [code lines](https://github.com/kubernetes/kubernetes/blob/481459d12dc82ab88e413886e2130c2a5e4a8ec4/pkg/scheduler/framework/runtime/framework.go#L857).
+
+## Summary
+
+During the tests, we found YuniKorn is performing really well, especially compared to the default scheduler. We have identified the major factors in YuniKorn where we can continue to improve the performance, and also explained why YuniKorn is performing better than the default scheduler. We also realized the limitations while scaling Kubernetes to thousands of nodes, that can be alleviated by using other techiques such as, e.g federation. At a result, YuniKorn is a high-efficient, high-throughput scheduler that perfectly positioned for running batch/mixed workloads on Kubernetes.
diff --git a/versioned_docs/version-1.5.0/performance/evaluate_perf_function_with_kwok.md b/versioned_docs/version-1.5.0/performance/evaluate_perf_function_with_kwok.md
new file mode 100644
index 0000000..a1825c8
--- /dev/null
+++ b/versioned_docs/version-1.5.0/performance/evaluate_perf_function_with_kwok.md
@@ -0,0 +1,173 @@
+---
+id: evaluate_perf_function_with_kwok
+title: Evaluate YuniKorn Performance with KWOK
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+[KWOK](https://kwok.sigs.k8s.io/) is a powerful toolkit designed to swiftly establish a cluster of thousands of Nodes within seconds. This page is dedicated to leveraging KWOK for performance testing. The objective is to conduct a comparative analysis between YuniKorn (Standard mode/Plugin mode) and Kube-scheduler, evaluating their capabilities in workload handling.
+
+KWOK conserves memory by emulating only node and pod behavior, unlike Kubemark, which emulates a kubelet and consumes a significant amount of memory.
+
+Using Kwok, we replicated previous performance tests, deploying 10 deployments on 5000 nodes with 5000 replicas each, resulting in a remarkably low total memory usage of 30-40GB. As a result, Kwok enables us to scale up experiments and assess the performance of YuniKorn without the need to consider bandwidth and Kubelet processing speed.
+
+## Environment
+
+The test is conducted using KWOK in a Cluster. The cluster environment is optimized according to the performance tuning settings in the YuniKorn documentation. For more details, refer to the [Benchmarking Tutorial](performance/performance_tutorial.md#performance-tuning). You can conveniently set up Kwok in your Kubernetes Cluster by downloading the scripts we provide [here](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/kwok-perf-test/kwok-setup.sh).
+
+For data monitoring, Prometheus will be employed to gather metrics. We'll use count(kube_pod_status_scheduled_time{namespace="default"}) as an indicator of throughput.
+
+## Test Cases
+
+We will start a comparative analysis with kube-scheduler to evaluate the throughput of these two different schedulers. In addition, we will compare the performance differences when managing large numbers of Taints and Tolerations, configuring Affinity and Anti-Affinity settings, and handling PriorityClass jobs.
+
+:::important
+YuniKorn schedules pods according to the application. Bearing this in mind, our testing focuses on assessing the impact of various features like Taint & Tolerations, Affinity, and PriorityClass on performance. This requires assigning unique configurations to each pod to understand their effects accurately. We use shell scripts to configure a large number of pods efficiently, storing their configurations in a YAML file. These pods are then deployed as needed to evaluate the impact on system performance.
+
+During this process, we've identified that the primary constraint is the processing capability of the api-server. Our data indicates that it is capable of deploying 5,000 pods in just 38 seconds, achieving a throughput of 131.6 pods per second.
+:::
+
+### Throughtput
+In this experiment, we will use the following three test cases to measure the throughput and scheduling duration of different schedulers. Each application will be deployed at one-second intervals.
+
+#### Test Case:
+
+| Test Case | Applications | Tasks | Total Pods |
+| --------- | ------------ | ----- | ---------- |
+| #1 | 1 | 5000 | 5000 |
+| #2 | 5 | 1000 | 5000 |
+| #3 | 25 | 200 | 5000 |
+
+#### Result:
+
+| #1 | kube-scheduler | yunikorn | yunikorn plugin mode |
+| -------------------- | -------------- | -------- | -------------------- |
+| makespan | 99 | 6 | 99 |
+| throughput(pods/sec) | 50.5 | 833.3 | 50.5 |
+
+
+| #2 | kube-scheduler | yunikorn | yunikorn plugin mode |
+| -------------------- | -------------- | -------- | -------------------- |
+| makespan | 99 | 9 | 100 |
+| throughput(pods/sec) | 50.5 | 555.6 | 50 |
+
+
+| #3 | kube-scheduler | yunikorn | yunikorn plugin mode |
+| -------------------- | -------------- | -------- | -------------------- |
+| makespan | 99 | 32 | 100 |
+| throughput(pods/sec) | 50.5 | 156.3 | 50 |
+:::note
+Regarding the throughput of the YuniKorn Scheduler, the third test took the longest time. This can be attributed to the fact that the application is deployed only every second, thereby lengthening the makespan.
+:::
+
+
+### Taint & Tolerations
+
+In the Taint & Tolerations test case, each node is initially assigned a taint that corresponds to its numerical identifier. Following this, pods are randomly assigned tolerations based on the indexes of different nodes. To assess the impact on system performance, we will deploy a YAML file containing 5000 pods and monitor the outcome.
+
+```yaml
+kubectl taint nodes kwok-node-1 key-1=value-1:NoSchedule
+```
+
+```yaml
+tolerations:
+- key: key-$rand
+ operator: "Exists"
+```
+
+#### Result:
+
+| Taint & Tolerations | kube-scheduler | yunikorn | yunikorn plugin mode |
+| -------------------- | -------------- | -------- | -------------------- |
+| makespan | 99 | 36 | 99 |
+| throughput(pods/sec) | 50.5 | 138.9 | 50.5 |
+
+
+### Affinity & Non-Affinity
+
+In the test cases for Affinity & Non-Affinity, 5,000 pods will be divided into four different combinations as follows:
+
+| Types of Node affinity and anti-affinity | Operator | Numbers of Pods |
+| ---------------------------------------- | -------- | --------------- |
+| Preferred | In | 625 |
+| Preferred | NotIn | 625 |
+| Required | In | 625 |
+| Required | NotIn | 625 |
+
+
+| Types of Pod affinity and anti-affinity | Operator | Numbers of Pods |
+| --------------------------------------- | -------- | --------------- |
+| Preferred | In | 1250 |
+| Preferred | NotIn | 1250 |
+
+
+In the Node affinity section, the matchExpressions value is set to the hostname of a randomly selected node.
+```yaml
+affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: $operator
+ values:
+ - kwok-node-$randHost
+```
+
+In the Pod affinity section, the value of matchExpressions is a randomly assigned applicationId.
+```yaml
+affinity:
+ podAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: applicationId
+ operator: $operator
+ values:
+ - nginx-$randAppID1
+ topologyKey: kubernetes.io/role
+```
+
+#### Result
+
+| Affinity & Non-Affinity | kube-scheduler | yunikorn | yunikorn plugin mode |
+| ----------------------- | -------------- | -------- | -------------------- |
+| makespan | 99 | 42 | 100 |
+| throughput(pods/sec) | 50.5 | 119 | 50 |
+
+
+
+### PriorityClass
+Firstly, deploy 100 distinct PriorityClasses. Subsequently, deploy Jobs with 5,000 Pods at the same time. As YuniKorn sorts priorities solely based on the application and queue, we will assign unique applicationIDs to each Pod and randomly assign different PriorityClasses to the Pods. This approach allows us to observe and analyze the throughput differences between different schedulers.
+
+#### Result
+
+| PriorityClass | kube-scheduler | yunikorn | yunikorn plugin mode |
+| -------------------- | -------------- | -------- | -------------------- |
+| makespan | 99 | 38 | 99 |
+| throughput(pods/sec) | 50.5 | 131.6 | 50.5 |
+
+
+## Summary
+
+The test results reveal that YuniKorn demonstrates the highest throughput across all three tests. Both Kube-scheduler and YuniKorn plugin mode perform comparably.
+
+In tests involving Taint & Tolerations, Affinity & Anti-affinity, and PriorityClass, it's observed that despite numerous parameters and constraints in the deployment process, there's minimal impact on the final throughput and makespan. With the YuniKorn scheduler, a relatively lower throughput was noted, primarily due to the api-server becoming a bottleneck when deploying 5,000 pods simultaneously.
diff --git a/versioned_docs/version-1.5.0/performance/metrics.md b/versioned_docs/version-1.5.0/performance/metrics.md
new file mode 100644
index 0000000..610e2d0
--- /dev/null
+++ b/versioned_docs/version-1.5.0/performance/metrics.md
@@ -0,0 +1,105 @@
+---
+id: metrics
+title: Scheduler Metrics
+keywords:
+ - metrics
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn leverages [Prometheus](https://prometheus.io/) to record metrics. The metrics system keeps tracking of
+scheduler's critical execution paths, to reveal potential performance bottlenecks. Currently, there are three categories
+for these metrics:
+
+- scheduler: generic metrics of the scheduler, such as allocation latency, num of apps etc.
+- queue: each queue has its own metrics sub-system, tracking queue status.
+- event: record various changes of events in YuniKorn.
+
+all metrics are declared in `yunikorn` namespace.
+### Scheduler Metrics
+
+| Metrics Name | Metrics Type | Description |
+| --------------------- | ------------ | ------------ |
+| containerAllocation | Counter | Total number of attempts to allocate containers. State of the attempt includes `allocated`, `rejected`, `error`, `released`. Increase only. |
+| applicationSubmission | Counter | Total number of application submissions. State of the attempt includes `accepted` and `rejected`. Increase only. |
+| application | Gauge | Total number of applications. State of the application includes `running`, `completed` and `failed`. |
+| node | Gauge | Total number of nodes. State of the node includes `active` and `failed`. |
+| nodeResourceUsage | Gauge | Total resource usage of node, by resource name. |
+| schedulingLatency | Histogram | Latency of the main scheduling routine, in milliseconds. |
+| sortingLatency | Histogram | Latency of all nodes sorting, in milliseconds. |
+| tryNodeLatency | Histogram | Latency of node condition checks for container allocations, such as placement constraints, in milliseconds. |
+| tryPreemptionLatency | Histogram | Latency of preemption condition checks for container allocations, in milliseconds. |
+
+### Queue Metrics
+
+| Metrics Name | Metrics Type | Description |
+| ------------------------- | ------------- | ----------- |
+| appMetrics | Gauge | Queue application metrics. State of the application includes `running`, `accepted`, `rejected`, `failed`, `completed`, `allocated`, `released`. |
+| ResourceMetrics | Gauge | Queue resource metrics. State of the resource includes `guaranteed`, `max`, `allocated`, `pending`, `preempting`. |
+
+### Event Metrics
+
+| Metrics Name | Metrics Type | Description |
+| ------------------------ | ------------ | ----------- |
+| totalEventsCreated | Gauge | Total events created. |
+| totalEventsChanneled | Gauge | Total events channeled. |
+| totalEventsNotChanneled | Gauge | Total events not channeled. |
+| totalEventsProcessed | Gauge | Total events processed. |
+| totalEventsStored | Gauge | Total events stored. |
+| totalEventsNotStored | Gauge | Total events not stored. |
+| totalEventsCollected | Gauge | Total events collected. |
+
+## Access Metrics
+
+YuniKorn metrics are collected through Prometheus client library, and exposed via scheduler restful service.
+Once started, they can be accessed via endpoint http://localhost:9080/ws/v1/metrics.
+
+## Aggregate Metrics to Prometheus
+
+It's simple to setup a Prometheus server to grab YuniKorn metrics periodically. Follow these steps:
+
+- Setup Prometheus (read more from [Prometheus docs](https://prometheus.io/docs/prometheus/latest/installation/))
+
+- Configure Prometheus rules: a sample configuration
+
+```yaml
+global:
+ scrape_interval: 3s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'yunikorn'
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ['docker.for.mac.host.internal:9080']
+```
+
+- start Prometheus
+
+```shell script
+docker pull prom/prometheus:latest
+docker run -p 9090:9090 -v /path/to/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus
+```
+
+Use `docker.for.mac.host.internal` instead of `localhost` if you are running Prometheus in a local docker container
+on Mac OS. Once started, open Prometheus web UI: http://localhost:9090/graph. You'll see all available metrics from
+YuniKorn scheduler.
+
diff --git a/versioned_docs/version-1.5.0/performance/performance_tutorial.md b/versioned_docs/version-1.5.0/performance/performance_tutorial.md
new file mode 100644
index 0000000..51c607f
--- /dev/null
+++ b/versioned_docs/version-1.5.0/performance/performance_tutorial.md
@@ -0,0 +1,522 @@
+---
+id: performance_tutorial
+title: Benchmarking Tutorial
+keywords:
+ - performance
+ - tutorial
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Overview
+
+The YuniKorn community continues to optimize the performance of the scheduler, ensuring that YuniKorn satisfies the performance requirements of large-scale batch workloads. Thus, the community has built some useful tools for performance benchmarking that can be reused across releases. This document introduces all these tools and steps to run them.
+
+## Hardware
+
+Be aware that performance result is highly variable depending on the underlying hardware. All results published in the doc can only be used as references. We encourage each individual to run similar tests on their own environments in order to get a result based on your own hardware. This doc is just for demonstration purpose.
+
+A list of servers being used in this test are (Huge thanks to [National Taichung University of Education](http://www.ntcu.edu.tw/newweb/index.htm), [Kuan-Chou Lai](http://www.ntcu.edu.tw/kclai/) for providing these servers for running tests):
+
+| Manchine Type | CPU | Memory | Download/upload(Mbps) |
+| --------------------- | --- | ------ | --------------------- |
+| HP | 16 | 36G | 525.74/509.86 |
+| HP | 16 | 30G | 564.84/461.82 |
+| HP | 16 | 30G | 431.06/511.69 |
+| HP | 24 | 32G | 577.31/576.21 |
+| IBM blade H22 | 16 | 38G | 432.11/4.15 |
+| IBM blade H22 | 16 | 36G | 714.84/4.14 |
+| IBM blade H22 | 16 | 42G | 458.38/4.13 |
+| IBM blade H22 | 16 | 42G | 445.42/4.13 |
+| IBM blade H22 | 16 | 32G | 400.59/4.13 |
+| IBM blade H22 | 16 | 12G | 499.87/4.13 |
+| IBM blade H23 | 8 | 32G | 468.51/4.14 |
+| WS660T | 8 | 16G | 87.73/86.30 |
+| ASUSPRO D640MB_M640SA | 4 | 8G | 92.43/93.77 |
+| PRO E500 G6_WS720T | 16 | 8G | 90/87.18 |
+| WS E500 G6_WS720T | 8 | 40G | 92.61/89.78 |
+| E500 G5 | 8 | 8G | 91.34/85.84 |
+| WS E500 G5_WS690T | 12 | 16G | 92.2/93.76 |
+| WS E500 G5_WS690T | 8 | 32G | 91/89.41 |
+| WS E900 G4_SW980T | 80 | 512G | 89.24/87.97 |
+
+The following steps are needed for each server, otherwise the large scale testing may fail due to the limited number of users/processes/open-files.
+
+### 1. Set /etc/sysctl.conf
+```
+kernel.pid_max=400000
+fs.inotify.max_user_instances=50000
+fs.inotify.max_user_watches=52094
+```
+### 2. Set /etc/security/limits.conf
+
+```
+* soft nproc 4000000
+* hard nproc 4000000
+root soft nproc 4000000
+root hard nproc 4000000
+* soft nofile 50000
+* hard nofile 50000
+root soft nofile 50000
+root hard nofile 50000
+```
+---
+
+## Deploy workflow
+
+Before going into the details, here are the general steps used in our tests:
+
+- [Step 1](#Kubernetes): Properly configure Kubernetes API server and controller manager, then add worker nodes.
+- [Step 2](#Setup-Kubemark): Deploy hollow pods,which will simulate worker nodes, name hollow nodes. After all hollow nodes in ready status, we need to cordon all native nodes, which are physical presence in the cluster, not the simulated nodes, to avoid we allocated test workload pod to native nodes.
+- [Step 3](#Deploy-YuniKorn): Deploy YuniKorn using the Helm chart on the master node, and scale down the Deployment to 0 replica, and [modify the port](#Setup-Prometheus) in `prometheus.yml` to match the port of the service.
+- [Step 4](#Run-tests): Deploy 50k Nginx pods for testing, and the API server will create them. But since the YuniKorn scheduler Deployment has been scaled down to 0 replica, all Nginx pods will be stuck in pending.
+- [Step 5](../user_guide/troubleshooting.md#restart-the-scheduler): Scale up The YuniKorn Deployment back to 1 replica, and cordon the master node to avoid YuniKorn allocating Nginx pods there. In this step, YuniKorn will start collecting the metrics.
+- [Step 6](#Collect-and-Observe-YuniKorn-metrics): Observe the metrics exposed in Prometheus UI.
+---
+
+## Setup Kubemark
+
+[Kubemark](https://github.com/kubernetes/kubernetes/tree/master/test/kubemark) is a performance testing tool which allows users to run experiments on simulated clusters. The primary use case is the scalability testing. The basic idea is to run tens or hundreds of fake kubelet nodes on one physical node in order to simulate large scale clusters. In our tests, we leverage Kubemark to simulate up to a 4K-node cluster on less than 20 physical nodes.
+
+### 1. Build image
+
+##### Clone kubernetes repo, and build kubemark binary file
+
+```
+git clone https://github.com/kubernetes/kubernetes.git
+```
+```
+cd kubernetes
+```
+```
+KUBE_BUILD_PLATFORMS=linux/amd64 make kubemark GOFLAGS=-v GOGCFLAGS="-N -l"
+```
+
+##### Copy kubemark binary file to the image folder and build kubemark docker image
+
+```
+cp _output/bin/kubemark cluster/images/kubemark
+```
+```
+IMAGE_TAG=v1.XX.X make build
+```
+After this step, you can get the kubemark image which can simulate cluster node. You can upload it to Docker-Hub or just deploy it locally.
+
+### 2. Install Kubermark
+
+##### Create kubemark namespace
+
+```
+kubectl create ns kubemark
+```
+
+##### Create configmap
+
+```
+kubectl create configmap node-configmap -n kubemark --from-literal=content.type="test-cluster"
+```
+
+##### Create secret
+
+```
+kubectl create secret generic kubeconfig --type=Opaque --namespace=kubemark --from-file=kubelet.kubeconfig={kubeconfig_file_path} --from-file=kubeproxy.kubeconfig={kubeconfig_file_path}
+```
+### 3. Label node
+
+We need to label all native nodes, otherwise the scheduler might allocate hollow pods to other simulated hollow nodes. We can leverage Node selector in yaml to allocate hollow pods to native nodes.
+
+```
+kubectl label node {node name} tag=tagName
+```
+
+### 4. Deploy Kubemark
+
+The hollow-node.yaml is down below, there are some parameters we can configure.
+
+```
+apiVersion: v1
+kind: ReplicationController
+metadata:
+ name: hollow-node
+ namespace: kubemark
+spec:
+ replicas: 2000 # the node number you want to simulate
+ selector:
+ name: hollow-node
+ template:
+ metadata:
+ labels:
+ name: hollow-node
+ spec:
+ nodeSelector: # leverage label to allocate to native node
+ tag: tagName
+ initContainers:
+ - name: init-inotify-limit
+ image: docker.io/busybox:latest
+ imagePullPolicy: IfNotPresent
+ command: ['sysctl', '-w', 'fs.inotify.max_user_instances=200'] # set as same as max_user_instance in actual node
+ securityContext:
+ privileged: true
+ volumes:
+ - name: kubeconfig-volume
+ secret:
+ secretName: kubeconfig
+ - name: logs-volume
+ hostPath:
+ path: /var/log
+ containers:
+ - name: hollow-kubelet
+ image: 0yukali0/kubemark:1.20.10 # the kubemark image you build
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 4194
+ - containerPort: 10250
+ - containerPort: 10255
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ command:
+ - /kubemark
+ args:
+ - --morph=kubelet
+ - --name=$(NODE_NAME)
+ - --kubeconfig=/kubeconfig/kubelet.kubeconfig
+ - --alsologtostderr
+ - --v=2
+ volumeMounts:
+ - name: kubeconfig-volume
+ mountPath: /kubeconfig
+ readOnly: true
+ - name: logs-volume
+ mountPath: /var/log
+ resources:
+ requests: # the resource of hollow pod, can modify it.
+ cpu: 20m
+ memory: 50M
+ securityContext:
+ privileged: true
+ - name: hollow-proxy
+ image: 0yukali0/kubemark:1.20.10 # the kubemark image you build
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ command:
+ - /kubemark
+ args:
+ - --morph=proxy
+ - --name=$(NODE_NAME)
+ - --use-real-proxier=false
+ - --kubeconfig=/kubeconfig/kubeproxy.kubeconfig
+ - --alsologtostderr
+ - --v=2
+ volumeMounts:
+ - name: kubeconfig-volume
+ mountPath: /kubeconfig
+ readOnly: true
+ - name: logs-volume
+ mountPath: /var/log
+ resources: # the resource of hollow pod, can modify it.
+ requests:
+ cpu: 20m
+ memory: 50M
+ tolerations:
+ - effect: NoExecute
+ key: node.kubernetes.io/unreachable
+ operator: Exists
+ - effect: NoExecute
+ key: node.kubernetes.io/not-ready
+ operator: Exists
+```
+
+once done editing, apply it to the cluster:
+
+```
+kubectl apply -f hollow-node.yaml
+```
+
+---
+
+## Deploy YuniKorn
+
+#### Install YuniKorn with helm
+
+We can install YuniKorn with Helm, please refer to this [doc](https://yunikorn.apache.org/docs/#install).
+We need to tune some parameters based on the default configuration. We recommend to clone the [release repo](https://github.com/apache/yunikorn-release) and modify the parameters in `value.yaml`.
+
+```
+git clone https://github.com/apache/yunikorn-release.git
+cd helm-charts/yunikorn
+```
+
+#### Configuration
+
+The modifications in the `value.yaml` are:
+
+- increased memory/cpu resources for the scheduler pod
+- disabled the admission controller
+- set the app sorting policy to FAIR
+
+please see the changes below:
+
+```
+resources:
+ requests:
+ cpu: 14
+ memory: 16Gi
+ limits:
+ cpu: 14
+ memory: 16Gi
+```
+```
+embedAdmissionController: false
+```
+```
+configuration: |
+ partitions:
+ -
+ name: default
+ queues:
+ - name: root
+ submitacl: '*'
+ queues:
+ -
+ name: sandbox
+ properties:
+ application.sort.policy: fair
+```
+
+#### Install YuniKorn with local release repo
+
+```
+Helm install yunikorn . --namespace yunikorn
+```
+
+---
+
+## Setup Prometheus
+
+YuniKorn exposes its scheduling metrics via Prometheus. Thus, we need to set up a Prometheus server to collect these metrics.
+
+### 1. Download Prometheus release
+
+```
+wget https://github.com/prometheus/prometheus/releases/download/v2.30.3/prometheus-2.30.3.linux-amd64.tar.gz
+```
+```
+tar xvfz prometheus-*.tar.gz
+cd prometheus-*
+```
+
+### 2. Configure prometheus.yml
+
+```
+global:
+ scrape_interval: 3s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'yunikorn'
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ['docker.for.mac.host.internal:9080']
+ # 9080 is internal port, need port forward or modify 9080 to service's port
+```
+
+### 3. Launch Prometheus
+```
+./prometheus --config.file=prometheus.yml
+```
+
+---
+## Run tests
+
+Once the environment is setup, you are good to run workloads and collect results. YuniKorn community has some useful tools to run workloads and collect metrics, more details will be published here.
+
+### 1. Scenarios
+In performance tools, there are three types of tests and feedbacks.
+
+| Test type | Description | Diagram | Log |
+| --------------------- | ------------------------------------------------------------------------------------------------------------------------ | ------------- | ----------------------------- |
+| node fairness | Monitor node resource usage(allocated/capicity) with lots of pods requests | Exist | Exist |
+| thourghput | Measure schedulers' throughput by calculating how many pods are allocated per second based on the pod start time | Exist | None |
+
+### 2. Build tool
+The performance tool is available in [yunikorn release repo](https://github.com/apache/yunikorn-release.git),clone the repo to your workspace.
+```
+git clone https://github.com/apache/yunikorn-release.git
+```
+Build the tool:
+```
+cd yunikorn-release/perf-tools/
+make build
+cd target/perf-tools-bin
+```
+It will look like this.
+
+
+### 3. Set test configuration
+Before start tests, check configuration whether meet your except.
+Default output path is `/tmp`, you can modify `common.outputrootpath` to change it.
+If you set these fields with large number to cause timeout problem, increase value in `common.maxwaitseconds` to allow it.
+
+#### Throughput case
+
+| Field | Description |
+| --- | --- |
+| SchedulerNames | List of scheduler will run the test |
+| ShowNumOfLastTasks | Show metadata of last number of pods |
+| CleanUpDelayMs | Controll period to refresh deployments status and print log |
+| RequestConfigs | Set resource request and decide number of deployments and pods per deployment with `repeat` and `numPods` |
+
+In this case,yunikorn and default scheduler will sequentially separately create ten deployments which contains fifty pods.
+It will look like these.
+
+
+
+#### Node fairness case
+
+| Field | Description |
+| --- | --- |
+| SchedulerNames | List of schduler will run the test |
+| NumPodsPerNode | It equals that total pods divided by nodes |
+| AllocatePercentage | Allow how much percentage of allocatable resource is allowed to allocate |
+
+Total number of pods will be multiplication of number of ready nodes and `NumPodsPerNode`.
+In following figure, there are thirteen ready nodes and `NumPodsPerNode` is eighty.
+There will be one thousand fourty pods created.
+
+
+
+#### e2e perf case
+Its field is similar to throughput one but there is only scheduler in each case.
+
+
+
+### 4. Diagrams and logs
+```
+./perf-tools
+```
+It will show result log when each case finished.
+When tests finished, it will look like
+
+We can find result diagrams and logs in `common.outputrootpath` which is in conf.yaml.
+Related diagrams and logs will be like this.
+
+---
+
+## Collect and Observe YuniKorn metrics
+
+After Prometheus is launched, YuniKorn metrics can be easily collected. Here is the [docs](metrics.md) of YuniKorn metrics. YuniKorn tracks some key scheduling metrics which measure the latency of some critical scheduling paths. These metrics include:
+
+ - **scheduling_latency_seconds:** Latency of the main scheduling routine, in seconds.
+ - **app_sorting_latency_seconds**: Latency of all applications sorting, in seconds.
+ - **node_sorting_latency_seconds**: Latency of all nodes sorting, in seconds.
+ - **queue_sorting_latency_seconds**: Latency of all queues sorting, in seconds.
+ - **container_allocation_attempt_total**: Total number of attempts to allocate containers. State of the attempt includes `allocated`, `rejected`, `error`, `released`. Increase only.
+
+you can select and generate graph on Prometheus UI easily, such as:
+
+
+
+
+---
+
+## Performance Tuning
+
+### Kubernetes
+
+The default K8s setup has limited concurrent requests which limits the overall throughput of the cluster. In this section, we introduced a few parameters that need to be tuned up in order to increase the overall throughput of the cluster.
+
+#### kubeadm
+
+Set pod-network mask
+
+```
+kubeadm init --pod-network-cidr=10.244.0.0/8
+```
+
+#### CNI
+
+Modify CNI mask and resources.
+
+```
+ net-conf.json: |
+ {
+ "Network": "10.244.0.0/8",
+ "Backend": {
+ "Type": "vxlan"
+ }
+ }
+```
+```
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "200Mi"
+ limits:
+ cpu: "100m"
+ memory: "200Mi"
+```
+
+
+#### Api-Server
+
+In the Kubernetes API server, we need to modify two parameters: `max-mutating-requests-inflight` and `max-requests-inflight`. Those two parameters represent the API request bandwidth. Because we will generate a large amount of pod request, we need to increase those two parameters. Modify `/etc/kubernetes/manifest/kube-apiserver.yaml`:
+
+```
+--max-mutating-requests-inflight=3000
+--max-requests-inflight=3000
+```
+
+#### Controller-Manager
+
+In the Kubernetes controller manager, we need to increase the value of three parameters: `node-cidr-mask-size`, `kube-api-burst` and `kube-api-qps`. `kube-api-burst` and `kube-api-qps` control the server side request bandwidth. `node-cidr-mask-size` represents the node CIDR. it needs to be increased as well in order to scale up to thousands of nodes.
+
+
+Modify `/etc/kubernetes/manifest/kube-controller-manager.yaml`:
+
+```
+--node-cidr-mask-size=21 //log2(max number of pods in cluster)
+--kube-api-burst=3000
+--kube-api-qps=3000
+```
+
+#### kubelet
+
+In single worker node, we can run 110 pods as default. But to get higher node resource utilization, we need to add some parameters in Kubelet launch command, and restart it.
+
+Modify start arg in `/etc/systemd/system/kubelet.service.d/10-kubeadm.conf`, add `--max-Pods=300` behind the start arg and restart
+
+```
+systemctl daemon-reload
+systemctl restart kubelet
+```
+
+---
+
+## Summary
+
+With Kubemark and Prometheus, we can easily run benchmark testing, collect YuniKorn metrics and analyze the performance. This helps us to identify the performance bottleneck in the scheduler and further eliminate them. The YuniKorn community will continue to improve these tools in the future, and continue to gain more performance improvements.
diff --git a/versioned_docs/version-1.5.0/performance/profiling.md b/versioned_docs/version-1.5.0/performance/profiling.md
new file mode 100644
index 0000000..662b341
--- /dev/null
+++ b/versioned_docs/version-1.5.0/performance/profiling.md
@@ -0,0 +1,122 @@
+---
+id: profiling
+title: Profiling
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Use [pprof](https://github.com/google/pprof) to do CPU, Memory profiling can help you understand the runtime status of YuniKorn scheduler. Profiling instruments have been
+added to YuniKorn rest service, we can easily retrieve and analyze them from HTTP
+endpoints.
+
+## CPU profiling
+
+At this step, ensure you already have YuniKorn running, it can be either running from
+local via a `make run` command, or deployed as a pod running inside of K8s. Then run
+
+```
+go tool pprof http://localhost:9080/debug/pprof/profile
+```
+
+The profile data will be saved on local file system, once that is done, it enters into
+the interactive mode. Now you can run profiling commands, such as
+
+```
+(pprof) top
+Showing nodes accounting for 14380ms, 44.85% of 32060ms total
+Dropped 145 nodes (cum <= 160.30ms)
+Showing top 10 nodes out of 106
+ flat flat% sum% cum cum%
+ 2130ms 6.64% 6.64% 2130ms 6.64% __tsan_read
+ 1950ms 6.08% 12.73% 1950ms 6.08% __tsan::MetaMap::FreeRange
+ 1920ms 5.99% 18.71% 1920ms 5.99% __tsan::MetaMap::GetAndLock
+ 1900ms 5.93% 24.64% 1900ms 5.93% racecall
+ 1290ms 4.02% 28.67% 1290ms 4.02% __tsan_write
+ 1090ms 3.40% 32.06% 3270ms 10.20% runtime.mallocgc
+ 1080ms 3.37% 35.43% 1080ms 3.37% __tsan_func_enter
+ 1020ms 3.18% 38.62% 1120ms 3.49% runtime.scanobject
+ 1010ms 3.15% 41.77% 1010ms 3.15% runtime.nanotime
+ 990ms 3.09% 44.85% 990ms 3.09% __tsan::DenseSlabAlloc::Refill
+```
+
+you can type command such as `web` or `gif` to get a graph that helps you better
+understand the overall performance on critical code paths. You can get something
+like below:
+
+
+
+Note, in order to use these
+options, you need to install the virtualization tool `graphviz` first, if you are using Mac, simply run `brew install graphviz`, for more info please refer [here](https://graphviz.gitlab.io/).
+
+## Memory Profiling
+
+Similarly, you can run
+
+```
+go tool pprof http://localhost:9080/debug/pprof/heap
+```
+
+this will return a snapshot of current heap which allows us to check memory usage.
+Once it enters the interactive mode, you can run some useful commands. Such as
+top can list top memory consumption objects.
+```
+(pprof) top
+Showing nodes accounting for 83.58MB, 98.82% of 84.58MB total
+Showing top 10 nodes out of 86
+ flat flat% sum% cum cum%
+ 32MB 37.84% 37.84% 32MB 37.84% github.com/apache/yunikorn-core/pkg/cache.NewClusterInfo
+ 16MB 18.92% 56.75% 16MB 18.92% github.com/apache/yunikorn-core/pkg/rmproxy.NewRMProxy
+ 16MB 18.92% 75.67% 16MB 18.92% github.com/apache/yunikorn-core/pkg/scheduler.NewScheduler
+ 16MB 18.92% 94.59% 16MB 18.92% github.com/apache/yunikorn-k8shim/pkg/dispatcher.init.0.func1
+ 1.04MB 1.23% 95.81% 1.04MB 1.23% k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypeWithName
+ 0.52MB 0.61% 96.43% 0.52MB 0.61% github.com/gogo/protobuf/proto.RegisterType
+ 0.51MB 0.61% 97.04% 0.51MB 0.61% sync.(*Map).Store
+ 0.50MB 0.6% 97.63% 0.50MB 0.6% regexp.onePassCopy
+ 0.50MB 0.59% 98.23% 0.50MB 0.59% github.com/json-iterator/go.(*Iterator).ReadString
+ 0.50MB 0.59% 98.82% 0.50MB 0.59% text/template/parse.(*Tree).newText
+```
+
+you can also run `web`, `pdf` or `gif` command to get the graph for heap.
+
+## Download profiling samples and analyze it locally
+
+We have included essential go/go-tool binaries in scheduler docker image, you should be able to do some basic profiling
+analysis inside of the docker container. However, if you want to dig into some issues, it might be better to do the analysis
+locally. Then you need to copy the samples file to local environment first. The command to copy files is like following:
+
+```
+kubectl cp ${SCHEDULER_POD_NAME}:${SAMPLE_PATH_IN_DOCKER_CONTAINER} ${LOCAL_COPY_PATH}
+```
+
+for example
+
+```
+kubectl cp yunikorn-scheduler-cf8f8dd8-6szh5:/root/pprof/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz /Users/wyang/Downloads/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz
+```
+
+once you get the file in your local environment, then you can run the `pprof` command for analysis.
+
+```
+go tool pprof /Users/wyang/Downloads/pprof.k8s_yunikorn_scheduler.samples.cpu.001.pb.gz
+```
+
+## Resources
+
+* pprof Document https://github.com/google/pprof/tree/master/doc.
diff --git a/versioned_docs/version-1.5.0/user_guide/acls.md b/versioned_docs/version-1.5.0/user_guide/acls.md
new file mode 100644
index 0000000..c1fd7f2
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/acls.md
@@ -0,0 +1,119 @@
+---
+id: acls
+title: ACLs
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+:::info
+User information is passed to the core scheduler from the kubernetes shim using the methodology defined [here](usergroup_resolution)
+:::
+
+## Usage
+Access Control Lists are generic for YuniKorn.
+They can be used in multiple places in YuniKorn.
+The current use case is limited to queue ACLs.
+
+Access control lists give access to the users and groups that have been specified in the list.
+They do not provide the possibility to explicitly remove or deny access to the users and groups specified in the list.
+
+If there is no access control list is configured access is *denied* by default.
+
+## Syntax
+The access control list is defined as:
+```
+ACL ::= “*” | userlist [ “ “ grouplist ]
+userlist ::= “” | user { “,” user }
+grouplist ::= “” | group { “,” group }
+```
+
+This definition specifies a wildcard of * which results in access for everyone.
+
+If the user list is empty and the group list is empty nobody will have access.
+This deny all ACL has two possible representations:
+* an empty access control list. (implicit)
+* a single space. (explicit)
+
+## Example config
+
+### Simple examples
+An ACL that allows access to just the user `sue`
+```yaml
+ adminacl: sue
+```
+Nobody else will get access, this is just for `sue`.
+`john` and `bob` will be denied access.
+
+An ACL that allows access to the user `sue` and the members of the group `dev`.
+```yaml
+ adminacl: sue dev
+```
+The user `sue` gets access based on her explicit mention in the user part of the ACL.
+Even though she is not a member of the group `dev`. Her group membership is irrelevant.
+
+The user named `john` whom is a member of the group `dev` will be allowed access based on his group membership.
+A third user called `bob` whom is not mentioned explicitly and is not a member of the `dev` group will be denied access.
+
+An ACL that allows access to the members of the groups `dev` and `test`.
+```yaml
+ adminacl: " dev,test"
+```
+The ACL must start with a space to indicate that there is no user list.
+If the ACL is not correctly quoted the space is dropped by the yaml parser.
+Since the user list is empty none of the users will get access unless they are a member of either the `dev` or `test` group.
+
+Looking at the same three users as before:
+The user `sue` is not a member of either group and is denied access.
+The user named `john` whom is a member of the group `dev` will be allowed access based on his group membership.
+`bob` is not a member of the `dev` group but is a member of `test` and will be allowed access.
+
+### Escaping and quotation marks
+ACLs are currently implemented in the queue configuration which uses a yaml file.
+This places some limitations on the how to escape the values.
+Incorrectly quoted values will cause a yaml parse error or could lead to the incorrect interpretation of the value.
+
+The following points need to be taken into account:
+1. The wildcard entry must be quoted in the yaml config.
+1. A simple list of users with or without it being followed by a list of groups does not need quoting but may be quoted.
+1. An ACL without a user list and just one or more groups must be quoted to find the starting space:
+
+Correctly quoted ACL example
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: test
+ submitacl: "*"
+ adminacl: sue dev,test
+ - name: product
+ submitacl: " product"
+```
+
+## Access check
+The access check follows the pattern:
+* check if the ACL is the wildcard
+* check if the user is in the user list
+* check if any of the groups the user is a member of is part of the group list
+
+If a check matches the ACL allows access and checking is stopped.
+If none of the checks match the ACL denies access.
+
+## User and Group information
+For User & Group resolution, please follow instructions defined [here](usergroup_resolution)
diff --git a/versioned_docs/version-1.5.0/user_guide/deployment_modes.md b/versioned_docs/version-1.5.0/user_guide/deployment_modes.md
new file mode 100644
index 0000000..6864a9c
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/deployment_modes.md
@@ -0,0 +1,51 @@
+---
+id: deployment_modes
+title: Deployment Modes
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## YuniKorn deployment modes
+
+YuniKorn can be deployed in two different modes: standard and plugin. In standard mode, YuniKorn runs as a customized
+Kubernetes scheduler. In plugin mode, YuniKorn is implemented as a set of plugins on top of the default Kubernetes
+scheduling framework.
+
+In both cases, it is recommended to run the admission controller as well, as this will ensure that only a single
+scheduler is active within your Kubernetes cluster. In this mode, the default Kubernetes scheduler (which is always running)
+will be bypassed for all pods except YuniKorn itself.
+
+## Which version should I use?
+
+### Standard mode
+
+Standard mode is currently the default. It is stable, efficient, and very performant. It is well-suited for
+deployments where most if not all pods are leveraging the queueing features of YuniKorn.
+
+### Plugin mode
+
+Plugin mode is a new deployment model where the scheduler is implemented on top of the default Kubernetes scheduling
+logic, allowing for better compatibility with the default Kubernetes scheduler. It is well-suited for mixed
+workloads (traditional Kubernetes as well as queued applications).
+
+Plugin mode is currently very new and has therefore not yet reached the maturity level of standard mode.
+
+To activate plugin mode when deploying with Helm, set the variable `enableSchedulerPlugin` to `true`.
+
diff --git a/versioned_docs/version-1.5.0/user_guide/gang_scheduling.md b/versioned_docs/version-1.5.0/user_guide/gang_scheduling.md
new file mode 100644
index 0000000..1b8dd24
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/gang_scheduling.md
@@ -0,0 +1,297 @@
+---
+id: gang_scheduling
+title: Gang Scheduling
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## What is Gang Scheduling
+
+When Gang Scheduling is enabled, YuniKorn schedules the app only when
+the app’s minimal resource request can be satisfied. Otherwise, apps
+will be waiting in the queue. Apps are queued in hierarchy queues,
+with gang scheduling enabled, each resource queue is assigned with the
+maximum number of applications running concurrently with min resource guaranteed.
+
+
+
+## Enable Gang Scheduling
+
+There is no cluster-wide configuration needed to enable Gang Scheduling.
+The scheduler actively monitors the metadata of each app, if the app has included
+a valid taskGroups definition, it will be considered as gang scheduling desired.
+
+:::info Task Group
+A task group is a “gang” of tasks in an app, these tasks are having the same resource profile
+and the same placement constraints. They are considered as homogeneous requests that can be
+treated as the same kind in the scheduler.
+:::
+
+### Prerequisite
+
+For the queues which runs gang scheduling enabled applications, the queue sorting policy should be set to `FIFO`.
+To configure queue sorting policy, please refer to doc: [app sorting policies](sorting_policies.md#application-sorting).
+
+#### Why the `FIFO` sorting policy
+
+When Gang Scheduling is enabled, the scheduler proactively reserves resources
+for each application. If the queue sorting policy is not FIFO-based (StateAware is FIFO based sorting policy),
+the scheduler might reserve partial resources for each app and causing resource segmentation issues.
+
+#### Side effects of `StateAware` sorting policy
+
+We do not recommend using `StateAware`, even-though it is a FIFO based policy. A failure of the first pod or a long initialisation period of that pod could slow down the processing.
+This is specifically an issue with Spark jobs when the driver performs a lot of pre-processing before requesting the executors.
+The `StateAware` timeout in those cases would slow down processing to just one application per timeout.
+This in effect will overrule the gang reservation and cause slowdowns and excessive resource usage.
+`StateAware` sorting is **deprecated** in YuniKorn 1.5.0 and will be **removed** from YuniKorn 1.6.0.
+
+### App Configuration
+
+On Kubernetes, YuniKorn discovers apps by loading metadata from individual pod, the first pod of the app
+is required to include a full copy of app metadata. If the app does not have any notion about the first or second pod,
+then all pods are required to carry the same taskGroups info. Gang scheduling requires taskGroups definition,
+which can be specified via pod annotations. The required fields are:
+
+| Annotation | Value |
+|------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| yunikorn.apache.org/task-group-name | Task group name, it must be unique within the application |
+| yunikorn.apache.org/task-groups | A list of task groups, each item contains all the info defined for the certain task group |
+| yunikorn.apache.org/schedulingPolicyParameters | Optional. A arbitrary key value pairs to define scheduling policy parameters. Please read [schedulingPolicyParameters section](#scheduling-policy-parameters) |
+
+#### How many task groups needed?
+
+This depends on how many different types of pods this app requests from K8s. A task group is a “gang” of tasks in an app,
+these tasks are having the same resource profile and the same placement constraints. They are considered as homogeneous
+requests that can be treated as the same kind in the scheduler. Use Spark as an example, each job will need to have 2 task groups,
+one for the driver pod and the other one for the executor pods.
+
+#### How to define task groups?
+
+The task group definition is a copy of the app’s real pod definition, values for fields like resources, node-selector, toleration
+and affinity should be the same as the real pods. This is to ensure the scheduler can reserve resources with the
+exact correct pod specification.
+
+#### Scheduling Policy Parameters
+
+Scheduling policy related configurable parameters. Apply the parameters in the following format in pod's annotation:
+
+```yaml
+annotations:
+ yunikorn.apache.org/schedulingPolicyParameters: "PARAM1=VALUE1 PARAM2=VALUE2 ..."
+```
+
+Currently, the following parameters are supported:
+
+`placeholderTimeoutInSeconds`
+
+Default value: *15 minutes*.
+This parameter defines the reservation timeout for how long the scheduler should wait until giving up allocating all the placeholders.
+The timeout timer starts to tick when the scheduler *allocates the first placeholder pod*. This ensures if the scheduler
+could not schedule all the placeholder pods, it will eventually give up after a certain amount of time. So that the resources can be
+freed up and used by other apps. If non of the placeholders can be allocated, this timeout won't kick-in. To avoid the placeholder
+pods stuck forever, please refer to [troubleshooting](troubleshooting.md#gang-scheduling) for solutions.
+
+` gangSchedulingStyle`
+
+Valid values: *Soft*, *Hard*
+
+Default value: *Soft*.
+This parameter defines the fallback mechanism if the app encounters gang issues due to placeholder pod allocation.
+See more details in [Gang Scheduling styles](#gang-scheduling-styles) section
+
+More scheduling parameters will added in order to provide more flexibility while scheduling apps.
+
+#### Example
+
+The following example is a yaml file for a job. This job launches 2 pods and each pod sleeps 30 seconds.
+The notable change in the pod spec is *spec.template.metadata.annotations*, where we defined `yunikorn.apache.org/task-group-name`
+and `yunikorn.apache.org/task-groups`.
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: gang-scheduling-job-example
+spec:
+ completions: 2
+ parallelism: 2
+ template:
+ metadata:
+ labels:
+ app: sleep
+ applicationId: "gang-scheduling-job-example"
+ queue: root.sandbox
+ annotations:
+ yunikorn.apache.org/task-group-name: task-group-example
+ yunikorn.apache.org/task-groups: |-
+ [{
+ "name": "task-group-example",
+ "minMember": 2,
+ "minResource": {
+ "cpu": "100m",
+ "memory": "50M"
+ },
+ "nodeSelector": {},
+ "tolerations": [],
+ "affinity": {}
+ }]
+ spec:
+ schedulerName: yunikorn
+ restartPolicy: Never
+ containers:
+ - name: sleep30
+ image: "alpine:latest"
+ command: ["sleep", "30"]
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "50M"
+```
+
+When this job is submitted to Kubernetes, 2 pods will be created using the same template, and they all belong to one taskGroup:
+*“task-group-example”*. YuniKorn will create 2 placeholder pods, each uses the resources specified in the taskGroup definition.
+When all 2 placeholders are allocated, the scheduler will bind the the real 2 sleep pods using the spot reserved by the placeholders.
+
+You can add more than one taskGroups if necessary, each taskGroup is identified by the taskGroup name,
+it is required to map each real pod with a pre-defined taskGroup by setting the taskGroup name. Note,
+the task group name is only required to be unique within an application.
+
+### Enable Gang scheduling for Spark jobs
+
+Each Spark job runs 2 types of pods, driver and executor. Hence, we need to define 2 task groups for each job.
+The annotations for the driver pod looks like:
+
+```yaml
+Annotations:
+ yunikorn.apache.org/schedulingPolicyParameters: “placeholderTimeoutSeconds=30”
+ yunikorn.apache.org/taskGroupName: “spark-driver”
+ yunikorn.apache.org/taskGroup: “
+ TaskGroups: [
+ {
+ Name: “spark-driver”,
+ minMember: 1,
+ minResource: {
+ Cpu: 1,
+ Memory: 2Gi
+ },
+ Node-selector: ...,
+ Tolerations: ...,
+ Affinity: ...
+ },
+ {
+ Name: “spark-executor”,
+ minMember: 10,
+ minResource: {
+ Cpu: 1,
+ Memory: 2Gi
+ }
+ }
+ ]
+ ”
+```
+
+:::note
+The TaskGroup resources must account for the memory overhead for Spark drivers and executors.
+See the [Spark documentation](https://spark.apache.org/docs/latest/configuration.html#application-properties) for details on how to calculate the values.
+:::
+
+For all the executor pods,
+
+```yaml
+Annotations:
+ # the taskGroup name should match to the names
+ # defined in the taskGroups field
+ yunikorn.apache.org/taskGroupName: “spark-executor”
+```
+
+Once the job is submitted to the scheduler, the job won’t be scheduled immediately.
+Instead, the scheduler will ensure it gets its minimal resources before actually starting the driver/executors.
+
+## Gang scheduling Styles
+
+There are 2 gang scheduling styles supported, Soft and Hard respectively. It can be configured per app-level to define how the app will behave in case the gang scheduling fails.
+
+- `Hard style`: when this style is used, we will have the initial behavior, more precisely if the application cannot be scheduled according to gang scheduling rules, and it times out, it will be marked as failed, without retrying to schedule it.
+- `Soft style`: when the app cannot be gang scheduled, it will fall back to the normal scheduling, and the non-gang scheduling strategy will be used to achieve the best-effort scheduling. When this happens, the app transits to the Resuming state and all the remaining placeholder pods will be cleaned up.
+
+**Default style used**: `Soft`
+
+**Enable a specific style**: the style can be changed by setting in the application definition the ‘gangSchedulingStyle’ parameter to Soft or Hard.
+
+#### Example
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: gang-app-timeout
+spec:
+ completions: 4
+ parallelism: 4
+ template:
+ metadata:
+ labels:
+ app: sleep
+ applicationId: gang-app-timeout
+ queue: fifo
+ annotations:
+ yunikorn.apache.org/task-group-name: sched-style
+ yunikorn.apache.org/schedulingPolicyParameters: "placeholderTimeoutInSeconds=60 gangSchedulingStyle=Hard"
+ yunikorn.apache.org/task-groups: |-
+ [{
+ "name": "sched-style",
+ "minMember": 4,
+ "minResource": {
+ "cpu": "1",
+ "memory": "1000M"
+ },
+ "nodeSelector": {},
+ "tolerations": [],
+ "affinity": {}
+ }]
+ spec:
+ schedulerName: yunikorn
+ restartPolicy: Never
+ containers:
+ - name: sleep30
+ image: "alpine:latest"
+ imagePullPolicy: "IfNotPresent"
+ command: ["sleep", "30"]
+ resources:
+ requests:
+ cpu: "1"
+ memory: "1000M"
+
+```
+
+## Verify Configuration
+
+To verify if the configuration has been done completely and correctly, check the following things:
+1. When an app is submitted, verify the expected number of placeholders are created by the scheduler.
+If you define 2 task groups, 1 with minMember 1 and the other with minMember 5, that means we are expecting 6 placeholder
+gets created once the job is submitted.
+2. Verify the placeholder spec is correct. Each placeholder needs to have the same info as the real pod in the same taskGroup.
+Check field including: namespace, pod resources, node-selector, toleration and affinity.
+3. Verify the placeholders can be allocated on correct type of nodes, and verify the real pods are started by replacing the placeholder pods.
+
+## Troubleshooting
+
+Please see the troubleshooting doc when gang scheduling is enabled [here](troubleshooting.md#gang-scheduling).
diff --git a/versioned_docs/version-1.5.0/user_guide/labels_and_annotations_in_yunikorn.md b/versioned_docs/version-1.5.0/user_guide/labels_and_annotations_in_yunikorn.md
new file mode 100644
index 0000000..550feae
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/labels_and_annotations_in_yunikorn.md
@@ -0,0 +1,55 @@
+---
+id: labels_and_annotations_in_yunikorn
+title: Labels and Annotations in YuniKorn
+---
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Labels and Annotations in YuniKorn
+YuniKorn utilizes several Kubernetes labels and annotations to support various features:
+
+### Labels in YuniKorn
+| Name | Description |
+|-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `applicationId` | Associates this pod with an application. |
+| `queue` | Selects the YuniKorn queue this application should be scheduled in. This may be ignored if a placement policy is in effect. |
+| `spark-app-selector` | Alternative method of specifying `applicationId` used by Spark Operator if the label `applicationId` and annotation `yunikorn.apache.org/app-id` unset. |
+| [DEPRECATED] `disableStateAware` | If present, disables the YuniKorn state-aware scheduling policy for this pod. Set internally by the YuniKorn admission controller. |
+| `placeholder` | Set if this pod represents a placeholder for gang scheduling. Set internally by YuniKorn. |
+
+### Annotations in YuniKorn
+All annotations are under the namespace `yunikorn.apache.org`. For example `yunikorn.apache.org/app-id`.
+
+| Name | Description |
+|-------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `app-id` | Assoiates this pod with an application.<br/>The priority of applicationID is determined by: annotation `yunikorn.apache.org/app-id` > label `applicationId` > label `spark-app-selector`. |
+| `queue` | Selects the YuniKorn queue this application should be scheduled in.<br/>The priority of queue is determined by: label `queue` > annotation `yunikorn.apache.org/queue` > default. |
+| `task-group-name` | Sets the task group name this pod belongs to for the purposes of gang scheduling. It must be listed within `task-groups`. |
+| `task-groups` | Defines the set of task groups for this application for gang scheduling. Each pod within an application must define all task groups. |
+| `schedulingPolicyParameters` | Arbitrary key-value pairs used to customize scheduling policies such as gang scheduling. |
+| `placeholder` | Set if this pod represents a placeholder for gang scheduling. Set internally by YuniKorn. |
+| `allow-preemption` | The `allow-preemption` annotation can be set on the Pod or PriorityClass object. The annotation in Pod takes priority over PriorityClass. It will trigger opt out of preemption for pods. Further details can be found in the [DaemonSet Scheduling using Simple Preemptor](./../design/simple_preemptor) documentation. |
+| `parentqueue` | Define a parent queue for a set of K8s namespaces. Further details can be found in the [ Resource Quota Management](resource_quota_management#parent-queue-mapping-for-namespaces) documentation. |
+| `namespace.quota` | Set the maximum capacity of the queue mapped to this namespace. Further details can be found in the [ Resource Quota Management](resource_quota_management#namespace-quota) documentation. |
+| [DEPRECATED] `namespace.max.cpu` | Replaced with ``namespace.quota`` since version 1.2.0 |
+| [DEPRECATED] `namespace.max.memory` | Replaced with `namespace.quota` since version 1.2.0 |
+| `namespace.enableYuniKorn` | Controls which namespaces will have pods forwarded to Yunikorn for scheduling. Further details can be found in the [Service Configuration #admission-controller-filtering-settings](service_config#admission-controller-filtering-settings)documentation. |
+| `namespace.generateAppId` | Controls which namespaces will have pods labeled with an `applicationId`. Further details can be found in the [Service Configuration #admission-controller-filtering-settings](service_config#admission-controller-filtering-settings) documentation. |
+
+For more details surrounding gang-scheduling labels and annotations, please refer to the documentation on [gang scheduling](user_guide/gang_scheduling.md).
diff --git a/versioned_docs/version-1.5.0/user_guide/placement_rules.md b/versioned_docs/version-1.5.0/user_guide/placement_rules.md
new file mode 100644
index 0000000..3b8b143
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/placement_rules.md
@@ -0,0 +1,355 @@
+---
+id: placement_rules
+title: App Placement Rules
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The basics for the placement rules are described in the [scheduler configuration design document](design/scheduler_configuration.md#placement-rules-definition).
+Multiple rules can be chained to form a placement policy.
+[Access control lists](user_guide/acls.md) and rule filters are defined per rule and enforced per rule.
+This document explains how to build a policy, including the rule usage, that is part of the scheduler with examples.
+
+## Configuration
+Rules are defined per partition as part of the scheduler queue configuration.
+The order that the rules are defined in is the order in which they are executed.
+If a rule matches the policy will stop executing the remaining rules.
+
+A matching rule generates a fully qualified queue name.
+This means that the name returned starts at the _root_ queue.
+There is no limit on the number of levels in the queue hierarchy that can be generated.
+
+When a rule is executed the result of rules that have been executed is unknown and not taken into account.
+Similar for rule that have not been executed yet: rules cannot influence other rules except when they are configured as the [parent](#parent-parameter) rule.
+
+If the policy does not generate a queue name and no more rules are left the application will be rejected.
+
+Basic structure for the rule placement definition in the configuration:
+```yaml
+placementrules:
+ - name: <name of the 1st rule>
+ - name: <name of the 2nd rule>
+```
+Each rule can take a predefined set of parameters in the configuration.
+The name of the rules that can be used are given in the [rule](#rules) description.
+A rule name is not case sensitive in the configuration.
+Rule name must follow the following naming convention:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _
+
+A rule that is not known, i.e. the name does not map to the rules defined below, will cause a initialisation error of the placement manager.
+Rules can also throw a parse error or an error during the initialisation if the parameters are incorrect.
+A rule set with an error can never become active.
+
+A placement manager is considered initialised if it has an active rule set.
+When the configuration is reloaded a new rule set will be created to replace the active rule set.
+In the case that a new rule set loaded contains an error, i.e. is broken, the placement manager ignores the new rule set.
+This means that the placement manager stays in a the state it was in when a broken rule set is loaded.
+If the placement manager keeps using the existing active rule set in the case that it was already initialised.
+A message will be logged about the broken and ignored configuration.
+
+Dots "." in the rule result are replaced by the string "\_dot_".
+A dot is replaced because it is used as the hierarchy separator in the fully qualified queue name.
+Replacing the dot occurs before the full queue hierarchy is build and the result is qualified.
+This means that we allow user name and or tag values to contain dots without the dots affecting the queue hierarchy.
+For queues in the configuration that as an example must map to username with a dot you must specify them as follows:
+A user rule with the user `user.name` will generate the queue name `root.user_dot_name` as output.
+If that "user queue" must be added to the configuration the `user_dot_name` name should be used.
+
+### Create parameter
+The create parameter is a boolean flag that defines if a queue that is generated by the rule may be created if it does not exist.
+There is no guarantee that the queue will be created because the existing queues might prevent the queue creation.
+If the queue generated by the rule does not exist and the flag is not set to _true_ the result of the rule will be a fail.
+
+Basic yaml entry for a rule with `create` flag:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ create: true
+```
+
+The default value is _false_.
+Allowed values: _true_ or _false_, any other value will cause a parse error.
+
+### Parent parameter
+The parent parameter allows specifying a rule that generates a parent queue for the current rule.
+Parent rules can be nested, a parent rule _may_ contain another parent rule.
+There is no enforced limit of parent rules that can be nested.
+
+A parent rule is treated as if it was a rule specified at the top level of the list and thus has the same parameters and requirements as a any other rule in the placement definition.
+The exception is that using a parent rule on a rule that already generates a fully qualified queue is considered a configuration error.
+This error can only occur on the rule of type [fixed](#fixed-rule), see the rule specification for more details.
+
+NOTE: the rule execution traverses down the list of parent rules and executes the last one in the list first.
+This means that the last parent rule will generate the queue directly below the root.
+See the examples for details.
+
+Basic yaml entry for a rule with a `parent` rule:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ parent:
+ name: <name of the parent rule>
+```
+
+The default value is _no_ parent.
+
+### Filter parameter
+The filter on a rule allows filtering the users that the rule applies to.
+A filter is a complex configuration object.
+
+The _users_ and _groups_ that can be configured can be one of two types:
+* a regular expression
+* a list of users or groups.
+
+If the entry for users or groups contains more than 1 entry in the yaml it is always considered a list of either users or groups.
+Duplicate entries in the lists are ignored and do not cause an error.
+Specifying a regular expression beside other list elements is not allowed.
+
+User and group names follow the standard linux user and group conventions.
+For user names:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _ . @ -
+* the last character may also be a $
+
+For group names:
+* start with a letter: a-z or A-Z
+* followed by 0 or more characters a-z, A-Z, 0-9 or _ -
+
+If the list is exactly one entry it can be either a single user or group or a regular expression.
+When an entry contains a character that is not allowed in a user or group name the entry is considered a regular expression.
+The regular expression must compile as specified.
+A regular expression that does not compile is ignored.
+
+Specifically for the group regular expression: matching is performed one group at a time not the against the list of groups.
+
+Basic `filter` yaml entry:
+```yaml
+filter:
+ type: deny
+ users:
+ - <user name or regexp>
+ - <user name>
+ groups:
+ - <group name or regexp>
+ - <group name>
+```
+
+The default value is _no_ filter.
+
+### Value parameter
+This is a generic value that can be used to pass to a rule to implement or alter its behaviour.
+The value It is used by the [fixed](#fixed-rule) and the [tag](#tag-rule) rule.
+The value is a single value in string form and is not interpreted or manipulated by the system.
+
+Basic yaml entry for a rule with a `value` set:
+```yaml
+placementrules:
+ - name: <name of the rule>
+ value: "any string"
+```
+
+The default is _no_ value.
+
+## Access Control List
+Access control lists are not defined in the rules but they impact the outcome of the placement policy.
+Two access control lists can be defined on a queue:
+1. Submit ACL: _submitacl_
+1. Administration ACL: _adminacl_
+
+The placement rule will only match if the ACL of the queue allows submit access via either ACL.
+The administrative queue ACL also provides _submit_ access.
+If the queue does not exist or does not have an ACL set, the ACL of the parent queue is checked.
+This recursive check is repeated until an ACL provides access or after the ACL of the root queue is checked.
+
+For more detail on the ACL syntax check the [ACL documentation](user_guide/acls.md).
+
+## Rules
+### Provided Rule
+Name to be used in the configuration: *provided*
+
+Returns the queue provided during the submission of the application.
+The behaviour of the this rule is to fully qualify the queue provided by the application if the queue is not fully qualified.
+If a parent rule is set and the queue provided in the application submission is fully qualified then the parent rule will not be executed.
+
+Supported parameters:
+* create
+* parent
+* filter
+
+Example: create the queue passed in by the user if it does not exist under the users name
+```yaml
+placementrules:
+ - name: provided
+ create: true
+ parent:
+ name: user
+ create: true
+```
+Application submit request by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.developer.my_special_queue` (parent rule set the user name)
+
+Application submit request by the user `developer`, queue in the application on submit: `root.dev_queue`<br/>
+Result: `root.dev_queue` (parent rule ignored)
+
+### User Name Rule
+Name to be used in the configuration: *user*
+
+Returns the queue based on the user name that is part of the submitted application.
+
+Supported parameters:
+* create
+* parent
+* filter
+
+Example: submit to a queue based on the user name, do not create the queue if the queue does not exist:
+```yaml
+placementrules:
+ - name: user
+ create: false
+```
+
+Application submit request by the user `finance.test`, queue does exist:<br/>
+Result: `root.finance_dot_test` (note the replacement of the dot)
+
+Application submit request by the user `developer`, queue does not exist:<br/>
+Result: failed, next rule executed
+
+### Fixed Rule
+Name to be used in the configuration: *fixed*
+
+Returns the name configured in the rule parameter _value_.
+The value configured must be a legal queue name or queue hierarchy.
+The name does not have to be a fully qualified queue name.
+The hierarchy in the name uses a dot as a separator for the queue names at the different levels in the hierarchy.
+The fixed rule can only fail if the queue configured does not exist and the create flag is not set as it will always return the configured queue.
+
+If the configured value is a fully qualified queue name configuring a parent rule will be considered an _error_.
+
+Supported parameters:
+* value (required)
+* create
+* parent
+* filter
+
+Example: a fixed queue returned always, without the `create` flag set and thus requires the queue to be defined as a leaf queue in the configuration.
+```yaml
+placementrules:
+ - name: fixed
+ value: last_resort
+```
+
+Application submit request by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.last_resort`
+
+### Tag Rule
+Name to be used in the configuration: *tag*
+
+Retrieves the queue name from the applications tags.
+The tag name that is checked for its value is configured in the rule using the `value`.
+Configuring a tag rule without a `value` set is an error, however an application does not have to have the tag set.
+
+If the tag is not set on the application the rule fails.
+If the tag value returned from the application is a fully qualified queue name the parent rule, if configured, will not be executed.
+
+Supported parameters:
+* value (required)
+* create
+* parent
+* filter
+
+Example: place an application based on the kubernetes `namespace` which gets sets automatically in the application when submitted:
+```yaml
+placementrules:
+ - name: tag
+ value: namespace
+ create: true
+```
+
+Application submit request for a kubernetes based application in the namespace `default` by the user `developer`, queue in the application on submit: `my_special_queue`<br/>
+Result: `root.default`
+
+Application submit request for a kubernetes based application in the namespace `testing` by the user `developer`<br/>
+Result: `root.testing`
+
+Application submit request for a non kubernetes based application by the user `developer`<br/>
+Result: failed, next rule executed
+
+## Complex examples
+In this complex example we chain three rules:
+1. a `user` rule, with a parent rule `tag` using the kubernetes namespace, to be used only for users that are part of and "dev" group.
+1. a `tag` rule using the kubernetes namespace, with a parent rule `fixed` using the existing queue `root.namespaces`.
+1. a `fixed` rule to place everything that reaches this point in the `root.default` queue.
+
+Example:
+```yaml
+placementrules:
+ - name: user
+ create: true
+ filter:
+ type: allow
+ groups:
+ - dev*
+ parent:
+ name: tag
+ value: namespace
+ - name: tag
+ value: namespace
+ create: true
+ parent:
+ name: fixed
+ value: root.namespaces
+ filter:
+ type: allow
+ users:
+ - john
+ - name: fixed
+ value: root.default
+```
+
+Application submit request for a kubernetes based application in the namespace `testing` by the user `john`<br/>
+Result: `root.namespaces.testing` (matched in rule 2)
+
+Application submit request for a kubernetes based application in the namespace `newapp` by the user `sarah` with groups membership `sarah, test_app, dev_app`<br/>
+Result: `root.newapp.sarah` (matched in rule 1)
+
+Application submit request for a kubernetes based application in the namespace `testapp` by the user `bob` with groups membership `bob`<br/>
+Result: `root.deault` (matched in rule 3)
+
+In this second example we chain two rules:
+1. a `fixed` rule to place everything in the `root.production` queue
+1. a `user` rule, with the create flag set
+
+In this case however we have ACLs set up on the `root.production` queue to only allow two specific user to use this queue.
+So even if the rule matches unless the user is either `john` or `bob` the application will not be placed in the `production` queue.
+All other users will match the second rule and use their own queue, which is created if it does not exist.
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: production
+ submitacl: john,bob
+ placementrules:
+ - name: fixed
+ value: root.production
+ - name: user
+ create: true
+```
diff --git a/versioned_docs/version-1.5.0/user_guide/preemption.md b/versioned_docs/version-1.5.0/user_guide/preemption.md
new file mode 100644
index 0000000..2492b18
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/preemption.md
@@ -0,0 +1,231 @@
+---
+id: preemption_cases
+title: Preemption
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+Preemption is an essential feature found in most schedulers, and it plays a crucial role in enabling key system functionalities like DaemonSets in K8s, as well as SLA and prioritization-based features.
+
+This document provides a brief introduction to the concepts and configuration methods of preemption in YuniKorn. For a more comprehensive understanding of YuniKorn's design and practical ideas related to preemption, please refer to the [design document](design/preemption.md).
+
+## Kubernetes Preemption
+
+Preemption in Kubernetes operates based on priorities. Starting from Kubernetes 1.14, you can configure preemption by adding a `preemptionPolicy` to the `PriorityClass`. However, it is important to note that preemption in Kubernetes is solely based on the priority of the pod during scheduling. The full documentation can be found [here](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#preemption).
+
+While Kubernetes does support preemption, it does have some limitations. Preemption in Kubernetes only occurs during the scheduling cycle and does not change once the scheduling is complete. However, when considering batch or data processing workloads, it becomes necessary to account for the possibility of opting out at runtime.
+
+## YuniKorn Preemption
+
+In Yunikorn, we offer two preemption types: general and DaemonSet. DaemonSet preemption is much more straightforward, as it ensures that pods which must run on a particular node are allowed to do so. The rest of the documentation only concerns generic preemption. For a comprehensive explanation of DaemonSet preemption, please consult the [design document](design/simple_preemptor.md).
+
+YuniKorn's generic preemption is based on a hierarchical queue model, enabling pods to opt out of running. Preemption is triggered after a specified delay, ensuring that each queue's resource usage reaches at least the guaranteed amount of resources. To configure the delay time for preemption triggering, you can utilize the `preemption.delay` property in the configuration.
+
+To prevent the occurrence of preemption storms or loops, where subsequent preemption tasks trigger additional preemption tasks, we have designed seven preemption laws. These laws are as follows:
+
+1. Preemption policies are strong suggestions, not guarantees
+2. Preemption can never leave a queue lower than its guaranteed capacity
+3. A task cannot preempt other tasks in the same application
+4. A task cannot trigger preemption unless its queue is under its guaranteed capacity
+5. A task cannot be preempted unless its queue is over its guaranteed capacity
+6. A task can only preempt a task with lower or equal priority
+7. A task cannot preempt tasks outside its preemption fence
+
+For a detailed explanation of these preemption laws, please refer to the preemption [design document](design/preemption.md#the-laws-of-preemption).
+
+Next, we will provide a few examples to help you understand the functionality and impact of preemption, allowing you to deploy it effectively in your environment. You can find the necessary files for the examples in the yunikorn-k8shim/deployment/example/preemption directory.
+
+Included in the files is a YuniKorn configuration that defines the queue configuration as follows:
+
+```bash
+queues.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: provided
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ properties:
+ preemption.policy: fence
+ preemption.delay: 10s
+ queues:
+ - name: 1-normal ...
+ - name: 2-no-guaranteed ...
+ - name: 3-priority-class ...
+ - name: 4-priority-queue ...
+ - name: 5-fence ...
+```
+
+Each queue corresponds to a different example, and the preemption will be triggered 10 seconds after deployment, as indicated in the configuration `preemption.delay: 10s`.
+
+### General Preemption Case
+
+In this case, we will demonstrate the outcome of triggering preemption when the queue resources are distributed unevenly in a general scenario.
+
+We will deploy 10 pods with a resource requirement of 1 to both `queue-1` and `queue-2`. First, we deploy to `queue-1` and then introduce a few seconds delay before deploying to `queue-2`. This ensures that the resource usage in `queue-1` will exceed that of `queue-2`, depleting all resources in the parent queue and triggering preemption.
+
+| Queue | Max Resource | Guaranteed Resource |
+| ---------------- | ------------ | ------------------- |
+| `normal` | 12 | - (not configured) |
+| `normal.queue-1` | 10 | 5 |
+| `normal.queue-2` | 10 | 5 |
+
+Result:
+
+When a set of guaranteed resources is defined, preemption aims to ensure that all queues satisfy their guaranteed resources. Preemption stops once the guaranteed resources are met (law 4). A queue may be preempted if it has more resources than its guaranteed amount. For instance, in this case, if queue-1 has fewer resources than its guaranteed amount (<5), it will not be preempted (law 5).
+
+| Queue | Resource before preemption | Resource after preemption |
+| ---------------- | -------------------------- | ------------------------- |
+| `normal.queue-1` | 10 (victim) | 7 |
+| `normal.queue-2` | 2 | 5 (guaranteed minimum) |
+
+
+
+### Priority
+
+In general, a pod can preempt a pod with equal or lower priority. You can set the priority by defining a [PriorityClass](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) or by utilizing [queue priorities](priorities).
+
+While preemption allows service-type pods to scale up or down through preemption, it can also lead to the preemption of pods that should not be preempted in certain scenarios:
+
+1. Spark Jobs, where the driver pod manages a large number of jobs, and if preempted, all jobs will be affected.
+2. Interactive pods, such as Python notebooks, have a significant impact when restarted and should be avoided from preemption.
+
+To address this issue, we have designed a "do not preempt me" flag. You can set the annotation `yunikorn.apache.org/allow-preemption` to `false` in the PriorityClass to prevent pod requests from being preempted.
+> **_NOTE:_** The flag `yunikorn.apache.org/allow-preemption` is a request only. It is not guaranteed but Pods annotated with this flag will be preempted last.
+
+
+### PriorityClass
+
+In this example, we will demonstrate the configuration of `yunikorn.apache.org/allow-preemption` using PriorityClass and observe its effect. The default value for this configuration is set to `true`.
+
+```bash
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+ name: preemption-not-allow
+ annotations:
+ "yunikorn.apache.org/allow-preemption": "false"
+value: 0
+```
+
+We will deploy 8 pods with a resource requirement of 1 to `queue-1`, `queue-2`, and `queue-3`, respectively. We will deploy to `queue-1` and `queue-2` first, followed by a few seconds delay before deploying to `queue-3`. This ensures that the resource usage in `queue-1` and `queue-2` will be greater than that in `queue-3`, depleting all resources in the parent queue and triggering preemption.
+
+| Queue | Max Resource | Guaranteed Resource | `allow-preemption` |
+| ------------ | ------------ | ------------------- | ------------------ |
+| `rt` | 16 | - | |
+| `rt.queue-1` | 8 | 3 | `true` |
+| `rt.queue-2` | 8 | 3 | `false` |
+| `rt.queue-3` | 8 | 3 | `true` |
+
+Result:
+
+When preemption is triggered, `queue-3` will start searching for a victim. However, since `queue-2` is set with allow-preemption as false, the resources of `queue-1` will be preempted.
+
+Please note that setting `yunikorn.apache.org/allow-preemption` is a strong recommendation but does not guarantee the lack of preemption. When this flag is set to `false`, it moves the Pod to the back of the preemption list, giving it a lower priority for preemption compared to other Pods. However, in certain scenarios, such as when no other preemption options are available, Pods with this flag may still be preempted.
+
+For example, even with `allow-preemption` set to `false`, DaemonSet pods can still trigger preemption. Additionally, if an application in `queue-1` has a higher priority than one in `queue-3`, the application in `queue-2` will be preempted because an application can never preempt another application with a higher priority. In such cases where no other preemption options exist, the `allow-preemption` flag may not prevent preemption.
+
+
+| Queue | Resource before preemption | Resource after preemption |
+| ------------ | -------------------------- | ------------------------- |
+| `rt.queue-1` | 8 (victim) | 5 |
+| `rt.queue-2` | 8 | 8 |
+| `rt.queue-3` | 0 | 3 (guaranteed minimum) |
+
+
+
+### Priority Queue
+
+In addition to utilizing the default PriorityClass in Kubernetes, you can configure priorities directly on a [YuniKorn queue](priorities).
+
+In the following example, we will demonstrate preemption based on queue priority.
+
+We will deploy five pods with a resource demand of 3 in the `high-pri` queue, `norm-pri` queue, and `low-pri` queue, respectively. We will deploy them to the `norm-pri` queue first, ensuring that the resources in the `root`(parent queue) will be fully utilized. This will result in uneven resource distribution among the queues, triggering preemption.
+
+| Queue | Max Resource | Guaranteed Resource | priority.offset |
+| --------------- | ------------ | ------------------- | --------------- |
+| `root` | 18 | - | |
+| `root.high-pri` | 10 | 6 | 100 |
+| `root.norm-pri` | 18 | 6 | 0 |
+| `root.low-pri` | 10 | 6 | -100 |
+
+Result:
+
+A queue with higher priority can preempt resources from a queue with lower priority, and preemption stops when the queue has preempted enough resources to satisfy its guaranteed resources.
+
+| Queue | Resource before preemption | Resource after preemption |
+| --------------- | -------------------------- | ------------------------- |
+| `root.high-pri` | 0 | 6 (guaranteed minimum) |
+| `root.norm-pri` | 18 (victim) | 12 |
+| `root.low-pri` | 0 | 0 |
+
+
+
+### Preemption Fence
+
+In a multi-tenant environment, it is essential to prevent one tenant from occupying the resources of another tenant. In YuniKorn, we map tenants to a queue hierarchy, the queue hierarchy can thus cross tenant boundaries.
+
+To address this issue, YuniKorn introduces a [preemption fence](design/preemption.md#preemption-fence), which is a setting on the queue that prevents preemption from looking at queues outside the fence boundary. The fence is a one-way fence. It prevents going out (i.e. higher up the queue hierarchy), but does not prevent coming in (or down) the queue hierarchy.
+
+```bash
+...
+queues:
+- name: default
+ properties:
+ preemption.policy: fence
+```
+
+We will use the following diagram as an example:
+
+
+
+In this example, we will sequentially deploy 15 pods with a resource requirement of 1 to each sub-queue.
+
+First, we deploy `queue-1` in `tenant-a` and wait until the application in `queue-1` occupies all the resources of `tenant-a`. Then, we deploy `queue-2` after the resources of `tenant-a` are fully utilized. Next, we deploy the application `ten-b.queue-3` and allocate resources to the system when the fence queue is full.
+
+| Queue | Max Resource | Guaranteed Resource | fence |
+| ------------------ | ------------ | ------------------- | ----- |
+| `rt` | 3 | - | true |
+| `rt.ten-a` | 15 | 5 | true |
+| `rt.ten-a.queue-1` | 15 | 2 | |
+| `rt.ten-a.queue-2` | 15 | 2 | true |
+| `rt.ten-b` | 15 | 10 | true |
+| `rt.ten-b.queue-3` | 15 | 10 | |
+| `rt.sys` | 15 | 10 | |
+
+Result:
+
+In this example, two imbalances are observed:
+
+- Within `ten-a`, `queue-1` occupies all the resources, while `queue-2` has no resources. However, since `queue-2` is configured with a fence, it cannot acquire resources from outside the fence.
+ 
+- Inside the `rt` queue, both `ten-a` and `ten-b` occupy all the resources, while the `sys` queue has no resources, and no fence is set up. Therefore, the `sys` queue can acquire resources from the queues in the hierarchy until its guaranteed resources are met. In this case, the `sys` queue acquires resources from both `ten-a` and `ten-b`.
+ 
+
+| Queue | Resource before preemption | Resource after preemption |
+| ------------------ | -------------------------- | ------------------------- |
+| `rt.ten-a` | 15 | 10 |
+| `rt.ten-a.queue-1` | 15 | 10 |
+| `rt.ten-a.queue-2` | 0 | 0 |
+| `rt.ten-b` | 15 | 10 |
+| `rt.ten-b.queue-3` | 15 | 10 |
+| `rt.sys` | 0 | 10 |
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/user_guide/priorities.md b/versioned_docs/version-1.5.0/user_guide/priorities.md
new file mode 100644
index 0000000..1137508
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/priorities.md
@@ -0,0 +1,220 @@
+---
+id: priorities
+title: App & Queue Priorities
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+YuniKorn has advanced support for priority scheduling. Priorities are
+specified on a per-task basis, but how those priorities are used can be
+customized for each queue.
+
+## Request Priority
+
+Every allocation request to the scheduler has a numeric priority associated
+with it. Any 32-bit integer value (positive or negative) may be used. Larger
+values indicate higher relative priorities.
+
+When using Kubernetes, priorities are defined in `PriorityClass`
+objects, which are referenced by `Pod` objects via a `priorityClassName`
+property. If no priority class is referenced, a `Pod` inherits the cluster
+default priority, typically `0`.
+
+See the Kubernetes [Pod Priority and Preemption](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/)
+documentation for more details.
+
+## Application Priority
+
+During scheduling, applications have a dynamic priority value which resolves
+to the highest priority outstanding request in that application. This allows
+the scheduler to dynamically reprioritize scheduling decisions.
+
+For example, if an application has two requests, one with priority `10` and
+another with priority `20`, the application's dynamic priority will be `20`.
+If the higher-priority request is satisfied, the application's priority will
+drop to `10`.
+
+When choosing between applications to schedule, the application sorting policy
+will (by default) schedule requests from higher-priority applications first.
+Priority can be ignored when sorting applications by setting the queue
+property `application.sort.priority` to `disabled` on a _leaf_ queue.
+
+## Queue Priority
+
+As with applications, queue priorities are also dynamically computed. For
+_parent_ queues, the queue's priority will be equal to the highest priority
+child queue it contains. For _leaf_ queues, the queue's priority will be
+equal to the highest priority application it contains.
+
+Queue priorities can also be adjusted automatically up or down by a fixed
+amount, specified in the `priority.offset` queue property. This can be useful
+in larger queue hierarchies to establish low or high priority queues.
+
+For example, if a _leaf_ queue with an offset of `5` contains two
+applications, one with priority `10` and another with priority `20`, the
+queue's priority will evaluate to `25` (`20 + 5`). If the higher-priority
+application no longer has requests, the queue's priority will drop to `15`
+(`10 + 5`).
+
+When choosing between child queues to schedule from, the queue sorting policy
+will (by default) schedule requests from higher-priority queues first.
+Priority can be ignored when sorting queues by setting the queue
+property `application.sort.priority` to `disabled` on a _parent_ queue.
+
+## Priority Fencing
+
+By default, priorities have global scope, meaning that higher-priority queues
+will be serviced before lower-priority queues regardless of their location
+in the queue hierarchy.
+
+However, it can be useful to limit prioritization to a subset of queues. This
+can be accomplished by setting the `priority.policy` queue property to
+`fence`. When a queue's priority is fenced, priorities are still evaluated
+within that queue (and subqueues), but the queue's priority itself will always
+evaluate to the `priority.offset` value or `0` if not specified.
+
+## Effective Priority
+
+Because of offsets and fencing, at any time a request may be thought of as
+having an _effective_ (or computed) priority based on its location within
+the queue hierarchy. Requests with higher effective priorities will be
+scheduled before those with lower effective priorities.
+
+## Examples
+
+### Single queue
+
+This example demonstrates a single leaf queue with all properties specified:
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: default
+ properties:
+ application.sort.policy: fifo
+ application.sort.priority: enabled
+ priority.policy: default
+ priority.offset: "0"
+```
+
+### Multitenancy
+
+This example demonstrates a complex queue tree containing multiple tenants
+with subqueues along with a multiple system queues:
+
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: system
+ queues:
+ - name: system-normal
+ properties:
+ priority.offset: "0"
+ - name: system-high
+ properties:
+ priority.offset: "1000"
+ - name: system-low
+ properties:
+ priority.offset: "-1000"
+ - name: tenants
+ properties:
+ priority.policy: "fence"
+ queues:
+ - name: tenant-a
+ properties:
+ priority.policy: "fence"
+ priority.offset: "10"
+ queues:
+ - name: child-a-1
+ - name: child-a-2
+ - name: tenant-b
+ properties:
+ priority.policy: "fence"
+ priority.offset: "0"
+ queues:
+ - name: child-b-1
+ - name: child-b-2
+
+```
+
+
+The `system-normal`, `system-high` and `system-low` queues are unfenced, and
+can therefore be prioritized above any other queue in the system. The
+`system-high` and `system-low` queues have offsets of `1000` and `-1000`
+respectively, so the priority of requests in those queues will be adjusted
+accordingly.
+
+The `tenants` _parent_ queue is priority-fenced, and has no `priority.offset`
+defined, so this queue will always be treated as though it has priority `0`.
+This ensures that normal and high-priority system tasks schedule ahead of
+tenant tasks, and low priority system tasks schedule after tenant tasks.
+
+The `tenant-a` and `tenant-b` _parent_ queues are also priority-fenced,
+preventing tenants from adjusting their priority relative to one another.
+The `tenant-a` queue also has a priority offset to ensure that it always
+schedules before `tenant-b`.
+
+The _leaf_ queues of `tenant-a` and `tenant-b` are not fenced, so tasks from
+the entire `tenant-a` or `tenant-b` subtree will prioritize relative to each
+other, but not outside their respective subtrees.
+
+
+
+In the figure above, multiple requests are shown with various priorities.
+Before scheduling, the queue priorities will be as follows:
+
+* root
+ * system: 1001
+ * system-normal: 10
+ * system-high: 1001
+ * system-low: -997
+ * tenants: 0 (fence)
+ * tenant-a: 10 (fence)
+ * child-a-1: 8
+ * child-a-2: 6
+ * tenant-b: 0 (fence)
+ * child-b-1: 9
+ * child-b-2: 8
+
+Queue traversal starts from the root, descending into each child queue in order
+starting with the highest effective priority. Assuming sufficient scheduling
+resources, the order of scheduling and effective queue priority changes are
+as follows:
+
+| Step | Queue | Task | Result |
+|------|---------------------------------|------|----------------------------------------------------------------------------------------|
+| 1 | root.system.system-high | P1 | **system-high**: `1001` -> n/a <br/> **system**: `1001` -> `10` |
+| 2 | root.system.system-normal | P10 | **system-normal**: `10` -> `2` <br/> **system**: `10` -> `2` |
+| 3 | root.system.system-normal | P2 | **system-normal**: `2` -> n/a <br/> **system**: `2` -> `-997` |
+| 4 | root.tenants.tenant-a.child-a-1 | P8 | **child-a-1**: `8` -> `5` |
+| 5 | root.tenants.tenant-a.child-a-2 | P6 | **child-a-2**: `6` -> `4` |
+| 6 | root.tenants.tenant-a.child-a-1 | P5 | **child-a-1**: `5` -> n/a |
+| 7 | root.tenants.tenant-a.child-a-2 | P4 | **child-a-2**: `4` -> n/a <br/> **tenant-a**: `10` -> n/a |
+| 8 | root.tenants.tenant-b.child-b-1 | P9 | **child-b-1**: `9` -> `7` |
+| 9 | root.tenants.tenant-b.child-b-2 | P8 | **child-b-2**: `8` -> n/a |
+| 10 | root.tenants.tenant-b.child-b-1 | P7 | **child-b-1**: `7` -> n/a <br/> **tenant-b**: `0` -> n/a <br/> **tenants**: `0` -> n/a |
+| 11 | root.system.system-low | P3 | **system-low**: `-997` -> n/a <br/> **system**: `-997` -> n/a |
+
diff --git a/versioned_docs/version-1.5.0/user_guide/prometheus.md b/versioned_docs/version-1.5.0/user_guide/prometheus.md
new file mode 100644
index 0000000..e75bdee
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/prometheus.md
@@ -0,0 +1,169 @@
+---
+id: prometheus
+title: Prometheus and Grafana
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+YuniKorn exposes its scheduling metrics via Prometheus. Thus, we need to set up a Prometheus server to collect these metrics.
+
+We will provide two methods for building Prometheus: either running it locally or using Helm to deploy it in your cluster. Additionally, in the Helm version, we will explain how to integrate it with Grafana and provide generic Grafana Dashboards for monitoring Yunikorn's metrics and observing the changes over time.
+
+If you don't know what metric can be used, you can use [REST API](../api/scheduler.md#metrics).
+
+## Run Prometheus locally
+
+### 1. Download Prometheus release
+
+```bash
+wget https://github.com/prometheus/prometheus/releases/download/v2.30.3/prometheus-2.30.3.linux-amd64.tar.gz
+```
+
+```bash
+tar xvfz prometheus-*.tar.gz
+cd prometheus-*
+```
+
+### 2. Configure prometheus.yml
+
+Prometheus collects metrics from *targets* by scraping metrics HTTP endpoints.
+
+```yaml
+global:
+ scrape_interval: 3s
+ evaluation_interval: 15s
+
+scrape_configs:
+ - job_name: 'yunikorn'
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ['localhost:9080']
+ # 9080 is internal port, need port forward or modify 9080 to service's port
+```
+
+### 3. Start port-forward
+
+Port forwarding for the core's web service on the standard port can be turned on via:
+
+```bash
+kubectl port-forward svc/yunikorn-service 9080:9080 -n yunikorn
+```
+
+`9080`is the default port for core's web service.
+
+### 4. Execute prometheus
+
+```bash
+./prometheus --config.file=prometheus.yml
+```
+
+
+
+### 5. Access the Prometheus UI
+
+You should be able to browse to a status page at [localhost:9090](http://localhost:9090/). Give it a couple of seconds to collect data about itself from its own HTTP metrics endpoint.
+
+
+
+You can also verify that Prometheus is serving metrics by navigating to its metrics endpoint:[localhost:9090/metrics](http://localhost:9090/metrics)
+
+## Deploy Prometheus and Grafana in a cluster.
+
+### 1. Add Prometheus repository to helm
+
+```yaml
+# add helm repo
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm repo update
+```
+
+### 2. Configuring yunikorn for prometheus
+
+Get the config from repository.
+```yaml
+helm show values prometheus-community/kube-prometheus-stack > /tmp/values.yaml
+```
+
+Add a new job in Prometheus to collect metrics by scraping the metrics HTTP endpoints of the targets.
+
+```yaml
+vim /tmp/values.yaml
+```
+```yaml
+...
+additionalScrapeConfigs:
+ - job_name: "yunikorn"
+ scrape_interval: 1s
+ metrics_path: '/ws/v1/metrics'
+ static_configs:
+ - targets: ["yunikorn-service.yunikorn.svc.cluster.local:9080"]
+...
+```
+
+### 3. Use helm to create Prometheus
+
+```yaml
+# create k8s namespace
+kubectl create namespace prometheus
+
+# deploy chart
+helm install prometheus prometheus-community/kube-prometheus-stack -n prometheus -f /tmp/values.yaml
+```
+
+### 4. Access the Prometheus Web UI
+
+```yaml
+kubectl port-forward -n prometheus svc/prometheus-kube-prometheus-prometheus 9090:9090
+```
+
+After running port-forward, you can enter [localhost:9090](http://localhost:9090) to access Prometheus Web UI.
+
+## Access Grafana Dashboard
+
+Port forwarding for the Grafana web service on the standard port can be turned on via:
+
+```yaml
+kubectl port-forward -n prometheus svc/prometheus-grafana 7070:80
+```
+
+After running port-forward, you can enter [localhost:7070](http://localhost:7070) to access grafana, and in the login page, enter account:`admin` ,password:`prom-operator`.
+
+
+
+### Download JSON files for Yunikorn Dashboard
+
+A dashboard consists of multiple panels that are organized and arranged in rows. Each panel has the ability to interact with data from any Grafana data source that has been configured. For more detailed information, please refer to the [Grafana Dashboards](https://grafana.com/docs/grafana/latest/dashboards).
+
+We provide a sample dashboard JSON file. To access it, you can navigate to the `/deployments/grafana-dashboard` directory in the Yunikorn-k8shim repository.
+
+You can refer to the [REST API](../api/scheduler.md#metrics) to build your own custom Dashboard.
+
+### Import the JSON files in the Dashboard
+
+Once you access the Dashboard page, you can proceed to import the provided JSON file.
+
+
+
+
+
+Once the import is complete, you will be able to locate Yunikorn's Dashboard on the page. From there, you can regularly monitor the status of Yunikorn.
+
+
diff --git a/versioned_docs/version-1.5.0/user_guide/queue_config.md b/versioned_docs/version-1.5.0/user_guide/queue_config.md
new file mode 100644
index 0000000..75bd241
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/queue_config.md
@@ -0,0 +1,463 @@
+---
+id: queue_config
+title: Partition and Queue Configuration
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The basis for the queue configuration is given in the [configuration design document](design/scheduler_configuration.md).
+
+This document provides the generic queue configuration.
+It references both the [Access Control Lists](user_guide/acls.md) and [Placement rules](user_guide/placement_rules.md) documentation.
+
+This document explains how to create the partition and queue configuration for the scheduler with examples.
+
+The scheduler relies on the shim to reliably provide user information as part of the application submission.
+The current shim identifies the user and the groups the user belongs to using the methodology provided in [User & Group Resolution](usergroup_resolution).
+
+## Configuration
+The configuration file for the scheduler that is described here only provides the configuration for the partitions and queues.
+
+By default the scheduler reads the ConfigMap section `queues.yaml` for partition and queue configuration. The section name can
+be changed by updating the `service.policyGroup` ConfigMap entry to be something other than `queues`.
+
+The example reference for the configuration is located in the scheduler core's [queues.yaml](https://github.com/apache/yunikorn-core/blob/master/config/queues.yaml) file.
+
+## Partitions
+Partitions are the top level of the scheduler configuration.
+There can be more than one partition defined in the configuration.
+
+Basic structure for the partition definition in the configuration:
+```yaml
+partitions:
+ - name: <name of the 1st partition>
+ - name: <name of the 2nd partition>
+```
+The default name for the partition is `default`.
+The partition definition contains the full configuration for the scheduler for a particular shim.
+Each shim uses its own unique partition.
+
+The partition must have at least the following keys defined:
+* name
+* [queues](#queues)
+
+The queues configuration is explained below.
+
+Optionally the following keys can be defined for a partition:
+* [placementrules](#placement-rules)
+* [limits](#limits)
+* nodesortpolicy
+* preemption
+
+Placement rules and limits are explained in their own chapters
+
+The `nodesortpolicy` key defines the way the nodes are sorted for the partition.
+Details on the values that can be used are in the [sorting policy](sorting_policies.md#node-sorting) documentation.
+
+The `preemption` key can have only one sub key: _enabled_.
+This boolean value defines the preemption behavior for the whole partition.
+
+The default value for _enabled_ is _true_.
+Allowed values: _true_ or _false_, any other value will cause a parse error.
+
+Example `partition` yaml entry with a `nodesortpolicy` of _fair_ and preemption disabled:
+```yaml
+partitions:
+ - name: <name of the partition>
+ nodesortpolicy: fair
+ preemption:
+ enabled: false
+```
+NOTE:
+Currently the Kubernetes unique shim does not support any other partition than the `default` partition..
+This has been logged as an [jira](https://issues.apache.org/jira/browse/YUNIKORN-22) for the shim.
+
+### Queues
+
+YuniKorn manages resources by leveraging resource queues. The resource queue has the following characters:
+- queues can have **hierarchical** structure
+- each queue can be preset with **min/max capacity** where min-capacity defines the guaranteed resource and the max-capacity defines the resource limit (aka resource quota)
+- tasks must be running under a certain leaf queue
+- queues can be **static** (loading from configuration file) or **dynamical** (internally managed by YuniKorn)
+- queue level **resource fairness is** enforced by the scheduler
+- a job can only run under a specific queue
+
+:::info
+The difference between YuniKorn queue and [Kubernetes namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/):
+Kubernetes namespace provides the scope for the Kubernetes resources, including the security context (i.e who can access the objects), and resource
+boundary when [resource quota](https://kubernetes.io/docs/concepts/policy/resource-quotas/) is defined (i.e how many resources can be used by the objects).
+On the other hand, YuniKorn queue is only used how many resources can be used by a group of jobs, and in which order. It provides
+a more fine-grained control on resource sharing across multiple tenants with considering of resource fairness, job ordering, etc. In most of the cases,
+YuniKorn queue can be used to replace the namespace resource quota, in order to provide more scheduling features.
+:::
+
+The _queues_ entry is the main configuration element.
+It defines a hierarchical structure for the queues.
+
+It can have a `root` queue defined but it is not a required element.
+If the `root` queue is not defined the configuration parsing will insert the root queue for consistency.
+The insertion of the root queue is triggered by:
+* If the configuration has more than one queue defined at the top level a root queue is inserted.
+* If there is only one queue defined at the top level and it is not called `root` a root queue is inserted.
+
+The defined queue or queues will become a child queue of the inserted `root` queue.
+
+Basic `queues` yaml entry with sub queue:
+```yaml
+queues:
+- name: <name of the queue>
+ queues:
+ - name: <name of the queue>
+```
+
+Supported parameters for the queues:
+* name
+* parent
+* queues
+* maxapplications
+* [properties](#properties)
+* adminacl
+* submitacl
+* [resources](#resources)
+* [limits](#limits)
+
+Each queue must have a _name_.
+The name of a queue must be unique at the level that the queue is defined.
+Since the queue structure is fully hierarchical queues at different points in the hierarchy may have the same name.
+As an example: the queue structure `root.testqueue` and `root.parent.testqueue` is a valid structure.
+A queue cannot contain a dot "." character as that character is used to separate the queues in the hierarchy.
+If the name is not unique for the queue in the configuration or contains a dot a parsing error is generated and the configuration is rejected.
+
+Queues in the structure will automatically get a type assigned.
+The type of the queue is based on the fact that the queue has children or sub queues.
+The two types of queues are:
+* parent
+* leaf
+
+Applications can only be assigned to a _leaf_ queue.
+A queue that has a child or sub queue in the configuration is automatically a _parent_ queue.
+If a queue does not have a sub the queue in the configuration it is a _leaf_ queue, unless the `parent` parameter is set to _true_.
+Trying to override a _parent_ queue type in the configuration will cause a parsing error of the configuration.
+
+Sub queues for a parent queue are defined under the `queues` entry.
+The `queues` entry is a recursive entry for a queue level and uses the exact same set of parameters.
+The `maxapplications` property is an integer value, larger than 1, which allows you to limit the number of running applications for the queue. Specifying a zero for `maxapplications` is not allowed as it would block all allocations for applications in the queue. The `maxapplications` value for a _child_ queue must be smaller or equal to the value for the _parent_ queue.
+
+The [properties](#properties) section contains simple key/value pairs. This is
+used for further queue customization of features such as
+[application sorting](sorting_policies.md#application-sorting) and priority
+scheduling. Future features will use the exisitng `properties` section as well
+to avoid the need to define a new structure for queue configuration.
+
+Access to a queue is set via the `adminacl` for administrative actions and for submitting an application via the `submitacl` entry.
+ACLs are documented in the [Access control lists](acls.md) document.
+
+Queue resource limits are set via the `resources` parameter.
+User and group limits are set via the `limits` parameter.
+As both entries are complex configuration entries they are explained in [resources](#resources) and [limits](#limits) below.
+
+An example configuration of a queue `root.namespaces` as a _parent_ queue with limits:
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: namespaces
+ parent: true
+ maxapplications: 12
+ resources:
+ guaranteed:
+ {memory: 1G, vcore: 10}
+ max:
+ {memory: 10G, vcore: 100}
+ queues:
+ - name: level1
+ maxapplications: 8
+ resources:
+ guaranteed:
+ {memory: 500M, vcore: 5}
+ max:
+ {memory: 5G, vcore: 50}
+```
+
+### Placement rules
+
+The placement rules are defined and documented in the [placement rule](placement_rules.md) document.
+
+Each partition can have only one set of placement rules defined.
+If no rules are defined the placement manager is not started and each application *must* have a queue set on submit.
+
+### Limits
+Limits define a set of limit objects for a queue, and can be set on a queue at any level.
+```yaml
+limits:
+ - limit: <description>
+ - limit: <description>
+```
+
+A limit object is a complex configuration object.
+It defines one limit for a set of users and or groups.
+Multiple independent limits can be set as part of one `limits` entry on a queue.
+Users and or groups that are not part of the limit setting will not be limited.
+
+A sample limits entry:
+```yaml
+limits:
+ - limit: <description>
+ users:
+ - <user name or "*">
+ - <user name>
+ groups:
+ - <group name or "*">
+ - <group name>
+ maxapplications: <1..maxint>
+ maxresources:
+ <resource name 1>: <0..maxint>[suffix]
+ <resource name 2>: <0..maxint>[suffix]
+```
+
+Limits are applied recursively in the case of a queue limit.
+This means that a limit on the `root` queue is an overall limit in the cluster for the user or group.
+A `root` queue limit is thus also equivalent with the `partition` limit.
+
+The limit object parameters:
+* limit
+* users
+* groups
+* maxapplications
+* maxresources
+
+The _limit_ parameter is an optional description of the limit entry.
+It is not used for anything but making the configuration understandable and readable.
+
+The _users_ and _groups_ that can be configured can be one of two types:
+* a star "*"
+* a list of users or groups.
+
+If the entry for users or groups contains more than one (1) entry it is always considered a list of either users or groups.
+The star "*" is the wildcard character and matches all users or groups.
+Duplicate entries in the lists are ignored and do not cause a parsing error.
+Specifying a star beside other list elements is not allowed.
+When a wildcard group is configured, a limit must be configured with at least one named group.
+Parsing will reject the configuration with limits that do not follow this rule.
+
+_maxapplications_ is an unsigned integer value, which allows you to limit the number of running applications for the configured user or group.
+Specifying 0 for _maxapplications_ is not allowed.
+
+The _maxresources_ parameter can be used to specify a limit for one or more resources.
+The _maxresources_ uses the same syntax as the [resources](#resources) parameter for the queue.
+Resources that are not specified in the list are not limited.
+A resource limit can be set to 0.
+This prevents the user or group from requesting the specified resource even though the queue or partition has that specific resource available.
+Specifying an overall resource limit of zero is not allowed.
+This means that at least one of the resources specified in the limit must be greater than zero.
+
+If a resource is not available on a queue the maximum resources on a queue definition should be used.
+Specifying a limit that is effectively zero, _maxapplications_ is zero and all resource limits are zero, is not allowed and will cause a parsing error.
+
+A limit is per user or group.
+It is *not* a combined limit for all the users or groups together.
+
+As an example:
+```yaml
+limit: "example entry"
+maxapplications: 10
+users:
+- sue
+- bob
+```
+In this case both the users `sue` and `bob` are allowed to run 10 applications.
+
+### Properties
+
+Additional queue configuration can be added via the `properties` section,
+specified as simple key/value pairs. The following parameters are currently
+supported:
+
+#### `application.sort.policy`
+
+Supported values: `fifo`, `fair`, `stateaware`
+
+Default value: `fifo`
+
+Sets the policy to be used when sorting applications within a queue. This
+setting has no effect on a _parent_ queue.
+
+**NOTE:** The `stateaware` policy is **deprecated** in YuniKorn 1.5.0 and will
+be **removed** from YuniKorn 1.6.0, where it will be treated as an alias for
+`fifo`.
+
+See the documentation on [application sorting](sorting_policies.md#application-sorting)
+for more information.
+
+
+#### `application.sort.priority`
+
+Supported values: `enabled`, `disabled`
+
+Default value: `enabled`
+
+When this property is `enabled`, priority will be considered when sorting
+queues and applications. Setting this value to `disabled` will ignore
+priorities when sorting. This setting can be specified on a _parent_ queue and
+will be inherited by _child_ queues.
+
+**NOTE:** YuniKorn releases prior to 1.2.0 did not support priorities when
+sorting. To keep the legacy behavior, set `application.sort.priority` to
+`disabled`.
+
+#### `priority.policy`
+
+Supported values: `default`, `fence`
+
+Default value: `default`
+
+Sets the inter-queue priority policy to use when scheduling requests.
+
+**NOTE**: This value is not inherited by child queues.
+
+By default, priority applies across queues globally. In other words,
+higher-priority requests will be satisfied prior to lower-priority requests
+regardless of which queue they exist within.
+
+When the `fence` policy is in use on a queue, the priorities of _child_ queues
+(in the case of a _parent_ queue) or applications (in the case of a _leaf_
+queue) will not be exposed outside the fence boundary.
+
+See the documentation on [priority support](priorities.md) for more information.
+
+#### `priority.offset`
+
+Supported values: any positive or negative 32-bit integer
+
+Default value: `0`
+
+Adjusts the priority of the queue relative to it's siblings. This can be useful
+to create high or low-priority queues without needing to set every task's
+priority manually.
+
+**NOTE**: This value is not inherited by child queues.
+
+When using the `default` priority policy, the queue's priority is adjusted up
+or down by this amount.
+
+When using the `fence` policy, the queue's priority is always set to the offset
+value (in other words, the priorities of tasks in the queue are ignored).
+
+See the documentation on [priority support](priorities.md) for more information.
+
+#### `preemption.policy`
+
+Supported values: `default`, `fence`, `disabled`
+
+Default value: `default`
+
+When using the `default` preemption policy, preemption is enabled for the queue.
+
+When using the [`fence` preemption policy](../design/preemption.md#preemption-fence), tasks running in or below the queue on which the property is set cannot preempt tasks outside the queue tree.
+
+When using the `disabled` preemption policy, tasks running within the queue can't be victims.
+
+#### `preemption.delay`
+
+Supported values: any positive [Golang duration string](https://pkg.go.dev/time#ParseDuration)
+
+Default value: `30s`
+
+The property can only be set on a leaf queue. A queue with pending requests can only trigger preemption after it has been in the queue for at least this duration.
+
+### Resources
+The resources entry for the queue can set the _guaranteed_ and or _maximum_ resources for a queue.
+Resource limits are checked recursively.
+The usage of a leaf queue is the sum of all assigned resources for that queue.
+The usage of a parent queue is the sum of the usage of all queues, leafs and parents, below the parent queue.
+
+The root queue, when defined, cannot have any resource limit set.
+If the root queue has any limit set a parsing error will occur.
+The maximum resource limit for the root queue is automatically equivalent to the cluster size.
+There is no guaranteed resource setting for the root queue.
+
+Maximum resources when configured place a hard limit on the size of all allocations that can be assigned to a queue at any point in time.
+A maximum resource can be set to 0 which makes the resource not available to the queue.
+Guaranteed resources are used in calculating the share of the queue and during allocation.
+It is used as one of the inputs for deciding which queue to give the allocation to.
+Preemption uses the _guaranteed_ resource of a queue as a base which a queue cannot go below.
+
+Basic `resources` yaml entry:
+```yaml
+resources:
+ guaranteed:
+ <resource name 1>: <0..maxint>[suffix]
+ <resource name 2>: <0..maxint>[suffix]
+ max:
+ <resource name 1>: <0..maxint>[suffix]
+ <resource name 2>: <0..maxint>[suffix]
+```
+Resources that are not specified in the list are not limited, for max resources, or guaranteed in the case of guaranteed resources.
+
+An optional suffix may be specified for resource quantities. Valid suffixes are `k`, `M`, `G`, `T`, `P`, and `E` for SI powers of 10,
+and `Ki`, `Mi`, `Gi`, `Ti`, `Pi`, and `Ei` for SI powers of 2. Additionally, resources of type `vcore` may have a suffix of `m` to indicate millicores. For example, `500m` is 50% of a vcore. Units of type `memory` are interpreted in bytes by default. All other resource types have no designated base unit.
+
+Note that this is a behavioral change as of YuniKorn 1.0. Prior versions interpreted `memory` as units of 1000000 (1 million) bytes and `vcore` as millicores.
+
+### Child Template
+
+The parent queue can provide a template to define the behaviour of dynamic leaf queues below it. A parent queue having no child template inherits the child template
+from its parent if that parent has one defined. A child template can be defined at any level in the queue hierarchy on a queue that is of the type parent.
+
+The supported configuration in template are shown below.
+1. application sort policy
+2. max resources
+3. guaranteed resources
+4. max applications
+
+As an example:
+```yaml
+ partitions:
+ - name: default
+ placementrules:
+ - name: provided
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ childtemplate:
+ maxapplications: 10
+ properties:
+ application.sort.policy: fifo
+ resources:
+ guaranteed:
+ vcore: 1
+ memory: 1G
+ max:
+ vcore: 20
+ memory: 600G
+ queues:
+ - name: parent
+ parent: true
+ childtemplate:
+ resources:
+ max:
+ vcore: 21
+ memory: 610G
+ - name: notemplate
+ parent: true
+```
+In this case, `root.parent.sales` will directly use the child template of parent queue `root.parent`. By contrast,
+`root.notemplate.sales` will use the child template set on the queue `root` since its parent queue `root.notemplate` inherits the child template from the queue `root`.
diff --git a/versioned_docs/version-1.5.0/user_guide/resource_quota_mgmt.md b/versioned_docs/version-1.5.0/user_guide/resource_quota_mgmt.md
new file mode 100644
index 0000000..a29bbbd
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/resource_quota_mgmt.md
@@ -0,0 +1,348 @@
+---
+id: resource_quota_management
+title: Resource Quota Management
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Quota configuration and rules
+YuniKorn can offer a finer grained resource quota management setup compared to the simple namespace resource quota provided by Kubernetes.
+
+On Kubernetes a pod must fit into the namespace quota when the pod is submitted.
+If the pod does not fit in the namespace quota the pod is rejected.
+The client must implement a retry-mechanism and re-submit the pod if it needs the pod to be scheduled.
+
+Contrary to quotas in Kubernetes YuniKorn does not enforce quotas on submission but only on actively consumed resources.
+To explain the difference: when using YuniKorn for quota enforcement a new pod submitted to Kubernetes is always accepted.
+Yunikorn will queue the pod without counting the queued pod's resources towards the consumed quota.
+When YuniKorn tries to schedule the pod it checks at scheduling time if the pod fits in the quota configured for the queue the pod is assigned to.
+If at that point the pod does not fit in the quota the pod is skipped and not counted in the resource consumption.
+This means that until a scheduling attempt of a pod is successful a pod it is not consuming resources in the YuniKorn quota system.
+
+Resource quotas in YuniKorn are linked to the queue and its place in the queue hierarchy.
+The base of the queue structure, the `root` queue, does not allow setting a quota as it reflects the current size of the cluster.
+Node additions and removals update the `root` queue quota automatically.
+
+Beside the `root` queue the quotas can be set, and is enforced, at any point in the hierarchy.
+Every queue can have a quota set. The quota is enforced recursively throughout the hierarchy.
+This means that a child queue can never use more resources than the **configured** quota of the parent queue.
+Setting a quota on a child queue larger than its parent queue's quota would thus not have any effect and is handled as a configuration error.
+
+In the hierarchy there are some further rules that need to be considered.
+If a parent queue has multiple children the sum of the **usage** of all children combined can never exceed the quota **configured** on the parent.
+However, from a configuration perspective this does not mean that the sum of the **configured** quotas for all children must be smaller than the parent quota.
+
+
+
+As an example the `root.parent` queue has a quota of 900.
+It contains three child queues, two with a quota set.
+The `root.parent.child1` has no quota set and will thus be limited to the `root.parent` quota.
+The two other queues `root.parent.child2` and `root.parent.child3` each have a quota of 750 set.
+During normal operation the total usage of the 3 child queues together will be 900.
+The applications running in each child queue have a demand of more than 1000 each.
+
+Distribution in that case could be any of:
+* all 900 used by just the `child1` queue
+* spread out evenly over the 3 queues (300 by each)
+* `child2` maxed out using 750, and the left over 150 used by `child3`
+
+The exact distribution between the queues will fluctuate and is dependent on the scheduling policies.
+
+## Converting Kubernetes resources and quotas
+Resource support for pods is limited to the resources specified as part of the _requests_ specification:
+* _cpu_ is mapped to _vcore_ with the value in milli cpu.
+* _memory_ is mapped to _memory_ with the value in MB (1 MB = 10^6 B = 1 000 000 B).
+* all other resources are mapped as provided.
+
+Extended resource as per the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) are supported.
+
+Example pod with a single container:
+```yaml
+apiVersion: v1
+kind: Pod
+spec:
+ containers:
+ - name: container-1
+ resources:
+ requests:
+ cpu: "250m"
+ memory: "1Gi"
+ hugepages-1Gi: "1"
+```
+The above specification will set pod resources request for scheduling in YuniKorn to:
+* _vcore_ -> 250m
+* _memory_ -> 1073741824
+* _hugepages-1Gi_ -> 1
+
+Two remarks:
+Multiple container specifications will be aggregated into one total pod resource request automatically. All memory is reported in bytes.
+
+In the case that static queue definitions are used for a queue there is no limit on the type of resource that can be specified in a quota.
+Quota annotations on namespaces, used as part of the automatic queue creation, are limited to the equivalent _cpu_ and _memory_ resources.
+See the [setup](#namespace-quota) below for the annotations on the namespace for quotas.
+
+## Kubernetes and YuniKorn quota interaction
+The recommendation is to turn off, not configure, the Kubernetes Namespace quotas.
+Using only YuniKorn queue quotas provides a more flexible setup and allows queueing of workloads.
+
+In a setup that has both YuniKorn and Kubernetes quotas turned on consider the following points:
+* Two separate configurations need to be maintained.
+ This increases the maintenance burden, and the possibility of configuration mistakes.
+* Both quotas will be enforced.
+
+Having both quotas turned on can lead to unexpected behaviour.
+The main issue is the fact that the Kubernetes namespace quota is enforced on submit.
+There are three combinations of quota configuration that are possible.
+The 3 combinations could have two effects when used in combination with the YuniKorn quota.
+
+1. Both quotas are _equal_: workloads will not be queued, the full configured quota can be used.
+ - Maximum usage and queueing will be limited to the set quota
+2. Kubernetes quota is _lower_ than YuniKorn: the YuniKorn quota will never be reached and workloads will not be queued.
+ - Maximum usage will be limited to the Kubernetes quota.
+3. Kubernetes quota is _higher_ than YuniKorn: YuniKorn will limit the usage to the quota set in YuniKorn.
+ The Kubernetes quota will be enforced on submit and thus set the limit for the workload that can be queued on top of the YuniKorn quota.
+ - Maximum usage will be limited to the YuniKorn quota.
+ - Workload queueing will be limited to the Kubernetes quota.
+
+:::note
+The following configuration examples are just to demonstrate the format needed
+to create a queue hierarchy with quotas set.
+:::
+
+## Static queue definition
+
+### Goal
+A preconfigured hierarchy of queues with a maximum and guaranteed capacity.
+The users can only submit applications to the leaf queues.
+This approach manages the resource capacity for each of the queues, which is suitable to the scenarios that queues do not change too often.
+
+### Configuration
+Apply the following configuration to YuniKorn's configmap to:
+* setup 3 queues under `root`
+* each queue has a specific guaranteed and maximum capacity
+* anyone can submit to any queue
+
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ submitacl: '*'
+ queues:
+ - name: advertisement
+ resources:
+ guaranteed:
+ memory: 500G
+ vcore: 50
+ max:
+ memory: 800G
+ vcore: 80
+ - name: search
+ resources:
+ guaranteed:
+ memory: 400G
+ vcore: 40
+ max:
+ memory: 600G
+ vcore: 60
+ - name: sandbox
+ resources:
+ guaranteed:
+ memory: 100G
+ vcore: 10
+ max:
+ memory: 100G
+ vcore: 10
+```
+
+### Run a workload
+In order to run applications in specific queues, you will need to set the following labels in all pod specs.
+All pods with the same `applicationID` label are considered ti be one application.
+In the below example the application `my-test-app` will run in the queue `root.sandbox`:
+
+```yaml
+labels:
+ app: my-test-app
+ applicationId: "my-test-app-01"
+ queue: root.sandbox
+```
+
+## Namespace to queue mapping
+
+### Goal
+Automatically map a Kubernetes `namespace` to a queue in YuniKorn.
+The user creates the required namespaces in Kubernetes.
+The YuniKorn k8s shim and core scheduler automatically pass the required information and map the namespace to a queue, creating the queue if it does not exist.
+The resource quota will be managed by YuniKorn instead of using the Kubernetes namespace quota.
+This does require the namespaces to be setup without Kubernetes quota enforcement and tags as per the [setup](#namespace-quota) below.
+
+### Configuration
+Apply the following configuration to YuniKorn's configmap:
+
+```yaml
+partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+ properties:
+ application.sort.policy: fifo
+```
+
+This configuration places an application based on the `tag` rule.
+The tag selected is the `namespace` tag which is automatically added by the k8s shim to all applications that get created.
+The `create` flag is set to true which will trigger the creation of the queue with the same name as the namespace if it does not exist.
+
+Applications within the automatically created child queues will be sorted based sorting policy set on the parent queue.
+In this case the property `application.sort.policy` is in this configuration set to `fifo`.
+
+You can change the configuration using the helm charts during the installation by overwriting the configuration in the
+[helm chart template](https://github.com/apache/yunikorn-release/blob/master/helm-charts/yunikorn/values.yaml#L71-L81).
+
+### Namespace quota
+Namespaces in Kubernetes contain the quota information.
+If a quota is set on a namespace Kubernetes will automatically enforce the quota.
+In the case that YuniKorn is used for quota enforcement no quota must be set on the namespace.
+
+To allow specifying a quota on the namespace the following annotations should be set in the namespace object:
+
+```yaml
+yunikorn.apache.org/namespace.quota: "{\"cpu\": \"64\", \"memory\": \"100G\", \"nvidia.com/gpu\": \"1\"}"
+```
+YuniKorn will parse these annotations and set the maximum capacity of the queue mapped to this namespace.
+The values specified follow the standard Kubernetes formatting and unit specification.
+Annotation value must be a single json compliant string. Ensure double quotes iare escaped properly to not cause any problems.
+
+The example above will limit the queue mapped to the annotated namespace to 64 CPUs, 100GB memory and 1 `nvidia.com/gpu`.
+
+[DEPRECATED]
+The below annotations are deprecated and will be removed from next major release.
+They only support mapping memory and cpu, not other resource types.
+```yaml
+yunikorn.apache.org/namespace.max.cpu: "64"
+yunikorn.apache.org/namespace.max.memory: "100Gi"
+```
+The example for the deprecated annotation will set the queue quota to 64 CPUs and 100GB memory.
+
+### Run a workload
+
+Applications, and the pods that are part of the application, can be submitted without specific labels.
+YuniKorn will automatically add the required tags.
+The configured placement rule will create the queue, if required, and add the application to the queue.
+
+For example, if an application is submitted to namespace `development`, then the application will run in the `root.development` queue.
+
+## Parent queue mapping for namespaces
+
+### Goal
+Though the tag placement rule using the `namespace` tag is capable of placing an application in a queue this might not be enough in all setups.
+In some cases, multi tenancy for example, namespaces need to be grouped together.
+Administrators could annotate namespaces which allows dynamic placement of applications based on multiple annotations if placement rules were setup.
+YuniKorn cannot and does not just add all annotations from a namespace to an application.
+
+To help support this grouping case a parent queue can be tagged on a namespace.
+
+### Configuration
+The configuration for this functionality consists of two pieces:
+1. the mapping rule
+1. the namespace annotation
+
+First we set the following configuration to YuniKorn's configmap:
+
+```yaml
+partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ parent:
+ name: tag
+ value: namespace.parentqueue
+ queues:
+ - name: root
+ queues:
+ - name: production
+ parent: true
+ - name: development
+ parent: true
+```
+
+The configuration used for the namespace to queue mapping is the same as [above](#namespace-to-queue-mapping).
+As an extension to the placement rule a `parent` rule is added to support the grouping.
+The parent rule is used to generate the parent, or the queue above, in the hierarchy.
+The rule uses the tag `namespace.parentqueue` from the application to generate the parent queue name.
+The `namespace.parentqueue` tag is automatically added by the Kubernetes shim but does require a namespace annotation (see below).
+
+In the example rule configuration given the `create` flag is not set on the parent rule.
+This means that the parent queue must exist in the configuration otherwise the application submit will fail.
+For the example configuration this means supported values for the parent are thus limited to `production` and `development`.
+
+Quotas cannot be set on the parent queue using any of these mappings.
+The quota linked to the namespace is set on the namespace queue not the parent as per the namespace mapping provided earlier.
+
+Parent queue quotas must always be set directly in the configuration.
+This requires the `create` flag to be set to `false` on the parent rule.
+
+### Namespace parent queue
+Contrary to the namespace name itself, and inline with the quota settings, the namespaces need to be annotated to use the parent queue mapping.
+Namespace names must be unique in Kubernetes which is not affected by this annotation.
+The same annotation value may be used for multiple namespaces:
+```yaml
+yunikorn.apache.org/parentqueue: root.production
+```
+
+The example annotation above will map the parent queue to the existing `root.production` queue.
+Note that the rule will fully qualify the name if needed, you can thus omit the `root.` part in the annotation.
+If the annotation starts with `root.` the system assumes it is a fully qualified queue name.
+
+To complete the picture here is an image that shows the mapping from Kubernetes namespaces to queues in YuniKorn.
+It uses the annotations on the namespaces in Kubernetes as described, and the example configuration for the mapping rules.
+The `finance` and `sales` namespaces become queues grouped under the parent queue `production`.
+The namespaces `dev` and `test` are placed under the `development` parent queue.
+
+
+
+### Run a workload
+Applications, and the pods that are part of the application, can be submitted without specific labels or changes.
+YuniKorn will add the tags, the placement rules will do the rest.
+The configured placement rule will create the queues, if required, and add the application to the queue.
+
+Since the namespace `finance` is annotated with the example value, and the rules are in place.
+Applications in the `finance` namespace will run in the `root.production.finance` queue that is created dynamically.
+
+### Application resource usage report
+The sum of the resources consumed by all the pods of an application is the total resource usage of the application.
+The resource usage of a pod is measured as `resource-unit times the runtime length of the pod`. For example, if the resource of a pod
+is described as `cpu: "100m" memory: "500M"`, then with a runtime of 30 seconds, the resource consumption will be
+`vcore: 3000, memory: 15000000000`.
+
+YuniKorn logs an application summary line with prefix "`YK_APP_SUMMARY:`" upon the application's completion. The summary includes various information about the application including the total resource usage. As an example:
+
+`YK_APP_SUMMARY: {"appId":"test-app-id","submissionTime":1685487033533,"startTime":1685487035535,"finishTime":1685488714483,"user":"nobody","queue":"root.test-queue","state":"Completed","rmID":"cluster-A","resourceUsage":{"instType1":{"memory":38315403247616,"vcore":1622000},"instType2":{"memory":368171374608384,"vcore":16245000}}}`
+
+For each instance type used by the application, a resource usage entry is reported in the summary. The resource usage is reported for both cpu and memory, as microcpu-seconds and byte-seconds respectively.
+
+As an example, this information could be ingested into a database table. Analysis then could be done to understand historically the needed resources for an application in general, and the requirements needed at a queue level for all applications scheduled to run in the queue. In addition, the resource usage could be translated to resource cost, and charged back to the application owner.
+
diff --git a/versioned_docs/version-1.5.0/user_guide/service_config.md b/versioned_docs/version-1.5.0/user_guide/service_config.md
new file mode 100644
index 0000000..2703ad1
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/service_config.md
@@ -0,0 +1,1088 @@
+---
+id: service_config
+title: Service Configuration
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The official distribution of YuniKorn is deployed via Helm charts to
+Kubernetes. Configuration for YuniKorn is split into two parts: Helm
+configuration, and YuniKorn service configuration.
+
+## Helm Configuration
+Helm configuration is used to configure options for the deployment of
+YuniKorn to Kubernetes.
+
+The following settings can be configured during YuniKorn installation
+via Helm, either via Helm's command-line, as in `--set key=value`, or
+via an external file: `-f file.yaml`. The examples below will be given in
+YAML syntax.
+
+### Container images
+YuniKorn ships as a set of container images. The locations and pull
+policies can be customized as follows:
+```yaml
+# Image information for the standard scheduler
+image:
+ repository: apache/yunikorn
+ tag: scheduler-1.0.0 # default depends on YuniKorn version
+ pullPolicy: Always
+
+# Image information for the plugin scheduler
+pluginImage:
+ repository: apache/yunikorn
+ tag: scheduler-plugin-1.0.0 # default depends on YuniKorn version
+ pullPolicy: Always
+
+# Image information for the web UI
+web:
+ image:
+ repository: apache/yunikorn
+ tag: web-1.0.0 # default depends on YuniKorn version
+ pullPolicy: Always
+
+# Image information for the admission controller
+admissionController:
+ image:
+ repository: apache/yunikorn
+ tag: admission-1.0.0 # default depends on YuniKorn version
+ pullPolicy: Always
+```
+### Kubernetes configuration
+
+#### affinity
+Sets the affinity for the YuniKorn scheduler pod.
+
+Default: `{}`
+
+Example:
+```yaml
+affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - primary1
+ - primary2
+```
+#### admissionController.affinity
+Sets the affinity for the YuniKorn admission controller pod.
+
+Default: `{}`
+
+Example:
+```yaml
+admissionController:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: In
+ values:
+ - primary1
+ - primary2
+```
+
+#### hostNetwork
+Controls whether the scheduler should run in the host network.
+
+Default: `false`
+
+Example:
+```yaml
+hostNetwork: true
+```
+
+#### admissionController.hostNetwork
+Controls whether the admission controller should run in the host network.
+
+Default: `true`
+
+Example:
+```yaml
+admissionController:
+ hostNetwork: false
+```
+#### imagePullSecrets
+Provides secrets needed for pulling YuniKorn images.
+
+Default: `[]`
+
+Example:
+```yaml
+imagePullSecrets:
+ - secret1
+ - secret2
+```
+#### nodeSelector
+Sets a node selector(s) to use for placement of the YuniKorn scheduler pod.
+
+Default: `{}`
+
+Example:
+```yaml
+nodeSelector:
+ node-role.kubernetes.io/control-plane: "true"
+```
+#### admissionController.nodeSelector
+Sets a node selector(s) to use for placement of the YuniKorn admission controller pod.
+
+Default: `{}`
+
+Example:
+```yaml
+admissionController:
+ nodeSelector:
+ node-role.kubernetes.io/control-plane: "true"
+```
+#### admissionController.replicaCount
+Sets the number of replicas to use for the YuniKorn admission controller. This
+can be set to greater than 1 for high-availability.
+
+Default: `1`
+
+Example:
+```yaml
+admissionController:
+ replicaCount: 2
+```
+#### serviceAccount
+Sets an alternate service account for the YuniKorn scheduler.
+
+Changing this value is not recommended, as Helm installs role-based access
+control (RBAC) policies for the default user that are required for proper
+functionaliy.
+
+Default: `yunikorn-admin`
+
+Example:
+```yaml
+serviceAccount: my-account
+```
+#### admissionController.serviceAccount
+Sets an alternate service account for the YuniKorn admission controller.
+
+Changing this value is not recommended, as Helm installs role-based access
+control (RBAC) policies for the default user that are required for proper
+functionaliy.
+
+Default: `yunikorn-admission-controller`
+
+Example:
+```yaml
+admissionController:
+ serviceAccount: my-account
+```
+#### service.type
+Sets the type of service used for the scheduler.
+
+Default: `ClusterIP`
+
+Example:
+```yaml
+service:
+ type: ClusterIP
+```
+#### admissionController.service.type
+Sets the type of service used for the admission controller.
+
+Default: `ClusterIP`
+
+Example:
+```yaml
+admissionController:
+ service:
+ type: ClusterIP
+```
+#### service.port
+Sets the port exposed in the YuniKorn scheduler service for the REST API.
+It is not recommended to change this value.
+
+Default: 9080
+
+Example:
+```yaml
+service:
+ port: 9080
+```
+#### service.portWeb
+Sets the port exposed in the YuniKorn scheduler service for the Web UI.
+It is not recommended to change this value.
+
+Default: 9889
+
+Example:
+```yaml
+service:
+ portWeb: 9889
+```
+#### tolerations
+Sets the tolerations for the YuniKorn scheduler pod.
+
+Default: `[]`
+
+Example:
+```yaml
+tolerations:
+ - key: node-role.kubernetes.io/control-plane
+ operator: Equal
+ value: "true"
+ effect: NoSchedule
+ - key: CriticalAddonsOnly
+ operator: Exists
+```
+#### admissionController.tolerations
+Sets the tolerations for the YuniKorn admission controller pod.
+
+Default: `[]`
+
+Example:
+```yaml
+admissionController:
+ tolerations:
+ - key: node-role.kubernetes.io/control-plane
+ operator: Equal
+ value: "true"
+ effect: NoSchedule
+ - key: CriticalAddonsOnly
+ operator: Exists
+```
+#### podLabels
+Sets the labels for the YuniKorn scheduler pod.
+
+Default: `{}`
+
+Example:
+```yaml
+podLabels:
+ app.kubernetes.io/name: scheduler
+ app.kubernetes.io/part-of: yunikorn
+```
+#### admissionController.podLabels
+Sets the labels for the YuniKorn admission controller pod.
+
+Default: `{}`
+
+Example:
+```yaml
+admissionController:
+ podLabels:
+ app.kubernetes.io/name: admission-controller
+ app.kubernetes.io/part-of: yunikorn
+```
+#### podAnnotations
+Sets the annotations for the YuniKorn scheduler pod.
+
+Default: `{}`
+
+Example:
+```yaml
+podAnnotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/path: /ws/v1/metrics
+ prometheus.io/port: 9080
+```
+#### admissionController.podAnnotations
+Sets the annotations for the YuniKorn admission controller pod.
+
+Default: `{}`
+
+Example:
+```yaml
+admissionController:
+ podAnnotations:
+ example.com/admission: "false"
+```
+### Resource utilization
+The resources requested for YuniKorn pods can be customized as follows:
+```yaml
+# Scheduler container resources
+resources:
+ requests:
+ cpu: 200m
+ memory: 1Gi
+ limits:
+ cpu: 4
+ memory: 2Gi
+
+# Web UI container resources
+web:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 100Mi
+ limits:
+ cpu: 100m
+ memory: 500Mi
+
+# Admission controller resources
+admissionController:
+ resources:
+ requests:
+ cpu: 100m
+ memory: 500Mi
+ limits:
+ cpu: 500m
+ memory: 500Mi
+```
+### Optional features
+
+#### embedAdmissionController
+Controls whether to enable the YuniKorn admission controller.
+
+Default: `true`
+
+Example:
+```yaml
+embedAdmissionController: false
+```
+#### enableSchedulerPlugin
+Controls whether to run YuniKorn in scheduler plugin mode.
+
+Default: `false`
+
+Example:
+```yaml
+enableSchedulerPlugin: true
+```
+### YuniKorn defaults
+
+#### yunikornDefaults
+Sets entries which will be rendered to the `yunikorn-defaults` ConfigMap. This
+can be used to pre-configure YuniKorn at deployment time. Any settings
+declared in [YuniKorn configuration](#yunikorn-configuration) may be set here.
+
+Default: `{}`
+
+Example:
+```yaml
+yunikornDefaults:
+ service.clusterId: yunikorn-01
+ service.policyGroup: group-01
+ group-01.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+```
+
+### Deprecated settings
+The following settings are deprecated, and will be removed from a future
+YuniKorn release. They should now be specified in the `yunikorn-configs` ConfigMap
+or via the Helm `yunikornDefaults` section:
+
+| Deprecated setting | ConfigMap replacement |
+|----------------------------------------|-------------------------------------------------|
+| operatorPlugins | - |
+| placeHolderImage | service.placeholderImage |
+| admissionController: processNamespaces | admissionController.filtering.processNamespaces |
+| admissionController: bypassNamespaces | admissionController.filtering.bypassNamespaces |
+| admissionController: labelNamespaces | admissionController.filtering.labelNamespaces |
+| admissionController: noLabelNamespaces | admissionController.filtering.noLabelNamespaces |
+| configuration | queues.yaml |
+
+Deprecated example:
+```yaml
+operatorPlugins: general
+placeHolderImage: registry.k8s.io/pause:3.7
+admissionController:
+ processNamespaces: "^spark-,^mpi-"
+ bypassNamespaces: "^kube-system$"
+ labelNamespaces: "^spark-"
+ noLabelNamespaces: "^mpi-legacy-"
+configuration: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+```
+Replacement example:
+```yaml
+yunikornDefaults:
+ service.policyGroup: queues
+ service.placeholderImage: registry.k8s.io/pause:3.7
+ admissionController.filtering.processNamespaces: "^spark-,^mpi-"
+ admissionController.filtering.bypassNamespaces: "^kube-system$"
+ admissionController.filtering.labelNamespaces: "^spark-"
+ admissionController.filtering.noLabelNamespaces: "^mpi-legacy-"
+ queues.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+```
+Currently, if both the deprecated parameter and the replacement ConfigMap entry are specified, the ConfigMap entry will take precedence.
+
+
+## YuniKorn Configuration
+
+Service configuration for YuniKorn is controlled by two Kubernetes ConfigMaps
+in the namespace where YuniKorn is installed: `yunikorn-defaults` and
+`yunikorn-configs`.
+
+At runtime, these ConfigMaps are polled by YuniKorn and merged together to form an
+effective configuration. If a setting is present in both ConfigMaps, the
+`yunikorn-configs` setting will override the one present in `yunikorn-defaults`.
+
+The purpose of `yunikorn-defaults` is to provide a mechanism for Helm to configure
+initial service configuration details. It should not be modified directly.
+
+The `yunikorn-configs` ConfigMap is completely unmanaged by Helm, and is meant for
+configurations which may change over time, such as queue configuration. All changes
+to YuniKorn configuration outside of provisioning infrastructure should be made here.
+
+### Default ConfigMap
+
+If neither ConfigMap is provided, or if an option is not specified, YuniKorn will
+use the default values listed here:
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: yunikorn-configs
+data:
+ service.clusterId: "mycluster"
+ service.policyGroup: "queues"
+ service.schedulingInterval: "1s"
+ service.volumeBindTimeout: "10s"
+ service.eventChannelCapacity: "1048576"
+ service.dispatchTimeout: "5m"
+ service.disableGangScheduling: "false"
+ service.enableConfigHotRefresh: "true"
+ service.placeholderImage: "registry.k8s.io/pause:3.7"
+ service.instanceTypeNodeLabelKey: "node.kubernetes.io/instance-type"
+ health.checkInterval: "30s"
+ log.level: "INFO"
+ kubernetes.qps: "1000"
+ kubernetes.burst: "1000"
+ admissionController.webHook.amServiceName: "yunikorn-admission-controller-service"
+ admissionController.webHook.schedulerServiceAddress: "yunikorn-service:9080"
+ admissionController.filtering.processNamespaces: ""
+ admissionController.filtering.bypassNamespaces: "^kube-system$"
+ admissionController.filtering.labelNamespaces: ""
+ admissionController.filtering.noLabelNamespaces: ""
+ admissionController.filtering.generateUniqueAppId: "false"
+ admissionController.filtering.defaultQueue: "root.default"
+ admissionController.accessControl.bypassAuth: "false"
+ admissionController.accessControl.trustControllers: "true"
+ admissionController.accessControl.systemUsers: "^system:serviceaccount:kube-system:"
+ admissionController.accessControl.externalUsers: ""
+ admissionController.accessControl.externalGroups: ""
+ queues.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+```
+### Service settings
+The following parameters are understood by YuniKorn:
+
+#### service.clusterId
+Sets an identifier for the cluster being configured. This is returned as part
+of REST API calls.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `mycluster`
+
+Example:
+```yaml
+service.clusterId: "yunikorn-east"
+```
+#### service.policyGroup
+Defines the policy group in use by this scheduler. The policy group is used to
+choose one of several queue configurations. The value of this setting plus an
+extension of `.yaml` controls the ConfigMap entry used to retrieve partition
+and queue configuration.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `queues`
+
+Example:
+```yaml
+service.policyGroup: group_b
+group_b.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: tag
+ value: namespace
+ create: true
+ queues:
+ - name: root
+ submitacl: '*'
+```
+
+#### service.schedulingInterval
+Controls the frequency with which YuniKorn executes scheduling runs.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `1s`
+
+Example:
+```yaml
+service.schedulingInterval: "5s"
+```
+#### service.volumeBindTimeout
+Controls the timeout before volume binding fails.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `10s`
+
+Example:
+```yaml
+service.volumeBindTimeout: "30s"
+```
+#### service.eventChannelCapacity
+Controls the number of internal scheduling events that YuniKorn will allow
+to be in-flight at one time. This acts as an out-of-memory guard.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `1048576`
+
+Example:
+```yaml
+service.eventChannelCapacity: "1000000"
+```
+#### service.dispatchTimeout
+Controls how long internal events will reattempt dispatching if the event
+channel is full. Warnings will be emitted if this timeout is exceeded.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `5m`
+
+Example:
+```yaml
+service.dispatchTimeout: "10m"
+```
+#### service.disableGangScheduling
+Allows global disabling of the gang scheduling feature (not recommended).
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `false`
+
+Example:
+```yaml
+service.disableGangScheduling: "true"
+```
+#### service.enableConfigHotRefresh
+Controls whether configuration should be hot-reloaded. By default, this
+is set to `true`, but it can be disabled to avoid changes to the
+ConfigMaps from being picked up until a scheduler restart.
+
+A change to this setting will be picked up without a restart of YuniKorn.
+
+NOTE: If this setting is disabled, it may not be re-enabled again without
+a restart of YuniKorn.
+
+Default: `true`
+
+Example:
+```yaml
+service.enableConfigHotRefresh: "false"
+```
+#### service.placeholderImage
+Sets the Pod image that will be used for gang scheduling placeholders.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `registry.k8s.io/pause:3.7`
+
+Example:
+```yaml
+service.placeholderImage: "registry.k8s.io/pause:3.6"
+```
+#### service.instanceTypeNodeLabelKey
+Sets the node label that will be used to determine the instance type of node.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `node.kubernetes.io/instance-type`
+
+Example:
+```yaml
+service.instanceTypeNodeLabelKey: "node.kubernetes.io/my-instance-type"
+```
+### Health settings
+
+#### health.checkInterval
+Sets the time between automatic health checks of YuniKorn.
+
+Setting the value to `0` or a negative interval will disable background health
+checking.
+
+A change to this setting will be picked up without a restart of YuniKorn.
+
+Default: `30s`
+
+Example:
+```yaml
+health.checkInterval: "1m"
+```
+### Log settings
+
+#### log.level
+Sets the default verbosity that YuniKorn will log at.
+
+A change to this setting will be picked up without a restart of YuniKorn. The available
+values can be numeric or textual:
+
+- `-1` / `debug` / `DEBUG`
+- `0` / `info` / `INFO`
+- `1` / `warn` / `WARN`
+- `2` / `error` / `ERROR`
+- `3` / `dpanic` / `DPANIC`
+- `4` / `panic` / `PANIC`
+- `5` / `fatal` / `FATAL`
+
+Default: `INFO`
+
+Example:
+```yaml
+log.level: "DEBUG"
+```
+
+#### log.{subsystem}.level
+Sets the verbosity that YuniKorn subsystem will log at.
+
+Yunikorn allows fine-grained logging configuration in a hierarchical manner. For example,
+setting an entry for `log.core.level` will configure all loggers that start with `core.`
+(including `core.scheduler`, etc.) unless a more specific configuration is present.
+Each subsystem[^1] has its log level.
+
+A change to this setting will be picked up without a restart of YuniKorn. The available
+values can be numeric or textual:
+
+- `-1` / `debug` / `DEBUG`
+- `0` / `info` / `INFO`
+- `1` / `warn` / `WARN`
+- `2` / `error` / `ERROR`
+- `3` / `dpanic` / `DPANIC`
+- `4` / `panic` / `PANIC`
+- `5` / `fatal` / `FATAL`
+
+Default: `INFO`
+
+Example:
+
+The `log.level` is the default log level for all loggers.
+
+```yaml
+log.level: "INFO"
+log.admission.level: "DEBUG"
+log.core.config.level: "INFO"
+```
+
+### Kubernetes settings
+
+#### kubernetes.qps
+Sets the number of Kubernetes queries per second (QPS) used by YuniKorn's
+Kubernetes client. This number must be >= 0.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `1000`
+
+Example:
+```yaml
+kubernetes.qps: "500"
+```
+#### kubernetes.burst
+Sets the maximum size of bursty queries to Kubernetes, temporarily allowing
+events to burst to this number while not still exceeding `kubernetes.qps`.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `1000`
+
+Example:
+```yaml
+kubernetes.burst: "500"
+```
+### Admission controller webhook settings
+
+#### admissionController.webHook.amServiceName
+Sets the name of the service that the YuniKorn admission controller is
+registered under. This is required for the admission controller to register
+itself properly with Kubernetes, and should normally not be changed.
+
+A change to this setting requires a restart of the YuniKorn admission controller
+to take effect.
+
+Default: `yunikorn-admission-controller-service`
+
+Example:
+```yaml
+admissionController.webHook.amServiceName: "yunikorn-admission-controller-alt-service-name"
+```
+#### admissionController.webHook.schedulerServiceAddress
+Sets the address of the YuniKorn scheduler service. This address must be
+reachable by the admission controller, and is used by the admission
+controller when validating ConfigMap changes. The admission controller
+will contact the REST API on the scheduler to validate any proposed
+ConfigMap changes. This setting should not normally be changed.
+
+A change to this setting requires a restart of the YuniKorn admission controller
+to take effect.
+
+Default: `yunikorn-service:9080`
+
+Example:
+```yaml
+admissionController.webHook.schedulerServiceAddress: "alt-yunikorn-service:9080"
+```
+### Admission controller filtering settings
+
+#### admissionController.filtering.processNamespaces
+Controls which namespaces will have pods forwarded to YuniKorn for scheduling.
+
+This setting is a comma-separated list of regular expressions. If this setting
+is an empty string, pods created in all namespaces will be scheduled by YuniKorn.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+```yaml
+# Schedule only pods in spark-* and mpi-* namespaces with YuniKorn
+admissionController.filtering.processNamespaces: "^spark-,^mpi-"
+```
+#### admissionController.filtering.bypassNamespaces
+Controls which namespaces will *not* have pods forwarded to YuniKorn for scheduling.
+This acts as an exception list to `admissionController.filtering.processNamespaces`.
+
+This setting is a comma-separated list of regular expressions. If this setting
+is an empty string, no pods in any namespaces will be excluded from processing by
+YuniKorn.
+
+By default, this setting excludes pods in the `kube-system` namespace as scheduling
+of these pods is often required for a node to be added to a cluster successfully.
+This could possibly prevent starting of YuniKorn itself or other critical services.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: `^kube-system$`
+
+Example:
+```yaml
+# Don't schedule pods in kube-system or fluentd-* namespaces
+admissionController.filtering.bypassNamespaces: "^kube-system$,^fluentd-"
+```
+
+> **_NOTE :_**
+> To simplify management, you can directly set the `yunikorn.apache.org/namespace.enableYunikorn` annotation on the namespace itself, regardless of whether it is specified in a regular expression. This annotation enables you to determine if the namespace should be managed by Yunikorn.
+
+#### admissionController.filtering.labelNamespaces
+Controls which namespaces will have pods labeled with an `applicationId`. By default,
+all pods which are scheduled by YuniKorn will have an `applicationId` label applied.
+
+When running YuniKorn using the standard deployment model, all pods should be labeled,
+as YuniKorn is unable to schedule pods without an `applicationId` defined.
+
+When running YuniKorn using the scheduler plugin deployment model, this setting can
+be used to filter which namespaces should be scheduled via YuniKorn's queueing model,
+and which should bypass queueing and be scheduled by the embedded default scheduler.
+
+This setting is a comma-separated list of regular expressions. If this setting
+is an empty string, all pods forwarded to YuniKorn will have an `applicationId` label
+applied.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+```yaml
+# Add applicationId labels to pods spark-* namespaces
+admissionController.filtering.labelNamespaces: "^spark-"
+```
+#### admissionController.filtering.noLabelNamespaces
+Controls which namespaces will *not* have pods labeled with an `applicationId`. This
+acts as an exception list to `admissionController.filtering.labelNamespaces`.
+
+When running YuniKorn using the standard deployment model, all pods should be labeled,
+as YuniKorn is unable to schedule pods without an `applicationId` defined.
+
+When running YuniKorn using the scheduler plugin deployment model, this setting can
+be used to filter which namespaces should be scheduled via YuniKorn's queueing model,
+and which should bypass queueing and be scheduled by the embedded default scheduler.
+
+This setting is a comma-separated list of regular expressions. If this setting
+is an empty string, no exclusions to `admissionController.filtering.labelNamespaces` will
+be applied.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+```yaml
+# Skip queueing in the noqueue namespace
+admissionController.filtering.labelNamespaces: "^noqueue$"
+```
+
+> **_NOTE :_**
+> To simplify management, you can directly set the `yunikorn.apache.org/namespace.generateAppId` annotation on the namespace itself, regardless of whether it is specified in a regular expression. This annotation enables you to determine if the namespace should be labeled by Yunikorn.
+
+#### admissionController.filtering.generateUniqueAppId
+YuniKorn generates `applicationId` for all the apps that do not have an `applicationId` to start with. This property controls if a *unique* `applicationId` should be generated for each such application or all the apps in a namespace should be bundled under a single `applicationId`.
+
+This setting is turned off by default and only one `applicationId` will be generated per namespace.
+
+When enabled, unique `applicationId` is generated using the namespace and the application's pod uid.
+
+Default: `false`
+
+Example:
+```yaml
+admissionController.filtering.generateUniqueAppId: "true"
+```
+
+#### admissionController.filtering.defaultQueue
+Controlls what will be the default queue name for the application.
+
+If the application does not define a queue name during app submission, admission controller will add a default queue name to the pod labels. `root.default` queue name will be added to the pod labels if this property is not set.
+
+In case, the default queue name needs to be updated to something other than `root.default`, `admissionController.filtering.defaultQueue` can be set with the desired queue name.
+
+Example:
+```yaml
+# Change default queue to root.mydefault
+admissionController.filtering.defaultQueue: "root.mydefault"
+```
+
+**_NOTE :_**
+The queue name needs to be a fully qualified queue name.
+
+For certain use-cases, there may be a need to skip adding a default queue name to the pod labels. In such cases, `admissionController.filtering.defaultQueue` can be set to empty string.
+
+Adding default queue name should be avoided when `provided` rule is used in conjunction with other placement rules and `provided` rule is higher in the hierarchy. If default queue label is added whenever there is no queue name specified, all the apps will be placed via `provided` rule and the other rules after that will never be executed.
+
+Default: `root.default`
+
+Example:
+```yaml
+# Skip adding default queue name
+admissionController.filtering.defaultQueue: ""
+```
+
+### Admission controller ACL settings
+
+#### admissionController.accessControl.bypassAuth
+Allow external users to create pods with user information already set.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: `false`
+
+Example:
+```yaml
+admissionController.accessControl.bypassAuth: "true"
+```
+#### admissionController.accessControl.trustControllers
+Allow controller users to create pods with user information already set.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: `true`
+
+Example:
+```yaml
+admissionController.accessControl.trustControllers: "false"
+```
+#### admissionController.accessControl.systemUsers
+Comma-separated list of regular expressions that match allowed controller
+service accounts.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: `^system:serviceaccount:kube-system:`
+
+Example:
+```yaml
+# allow all kube-system accounts as well as kube-controller-manager
+admissionController.accessControl.systemUsers: "^system:serviceaccount:kube-system,^system:kube-controller-manager$"
+```
+#### admissionController.accessControl.externalUsers
+Comma-separated list of regular expressions that match allowed external users.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+```yaml
+# allow 'alice', 'bob', and 'admin-*'
+admissionController.accessControl.externalUsers: "^alice$,^bob$,^admin-"
+```
+#### admissionController.accessControl.externalGroups
+Comma-separated list of regular expressions that match allowed external groups.
+
+A change to this setting will be picked up without a restart of the admission controller.
+
+Default: empty
+
+Example:
+```yaml
+# allow 'sales', 'marketing', and 'admin-*'
+admissionController.accessControl.externalGroups: "^sales$,^marketing$,^admin-"
+```
+
+### Using compressed values
+
+The data in ConfigMap cannot exceed 1 MiB. YuniKorn supports the gzip algorithm to decompress data in the `binaryData` field.
+If a key ends with `.gz`. YuniKorn will treat the value as gzip-compressed data and decompress it automatically. The base64 encoding is automatically.
+If a value is set in both the `data` and `binaryData` sections, the value in the `binaryData` section will be used.
+
+Example:
+
+Users can run the command to get the value.
+
+```bash
+echo "
+partitions:
+ - name: default
+ queues:
+ - name: root
+ submitacl: '*'
+ parent: true
+ queues:
+ - name: parent
+ submitacl: '*'" | gzip | base64
+```
+
+Set the result in the `binaryData` field.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: yunikorn-configs
+binaryData:
+ queues.yaml.gz: "H4sIAMyHs2UAA2WMSQ6AIBAE77yibyQmfoDfoI4JCYvCzP/FjWDsY3Wl1GYzO3YpFqOAEdEGMlhoteK5EmAXErrec6+RU+IHAUWm4NjO3kAPuuHapsgGnIUa/Ob65K13xy98AFwE9HmuAAAA"
+```
+
+### Deprecated settings
+
+#### service.operatorPlugins
+
+**_DEPRECATED in 1.4.0:_** No replacement
+
+Controls the set of operator plugins which are enabled within YuniKorn.
+Currently, only the `general` plugin is implemented, and the plugin
+functionality will be removed entirely in a future release. The `general`
+plugin should not be disabled as it is critical to the proper operation of
+YuniKorn.
+
+A change to this setting requires a restart of YuniKorn to take effect.
+
+Default: `general`
+
+Example:
+```yaml
+service.operatorPlugins: "general"
+```
+
+[^1]: Available log subsystem values:
+ - admission
+ - admission.client
+ - admission.conf
+ - admission.utils
+ - admission.webhook
+ - core
+ - core.config
+ - core.entrypoint
+ - core.events
+ - core.metrics
+ - core.opentracing
+ - core.resources
+ - core.rest
+ - core.rmproxy
+ - core.rpc
+ - core.scheduler
+ - core.scheduler.allocation
+ - core.scheduler.application
+ - core.scheduler.application.usage
+ - core.scheduler.context
+ - core.scheduler.fsm
+ - core.scheduler.health
+ - core.scheduler.node
+ - core.scheduler.partition
+ - core.scheduler.preemption
+ - core.scheduler.queue
+ - core.scheduler.reservation
+ - core.scheduler.ugm
+ - core.security
+ - core.utils
+ - deprecation
+ - kubernetes
+ - shim
+ - shim.appmgmt
+ - shim.appmgmt.general
+ - shim.appmgmt.sparkoperator
+ - shim.cache.application
+ - shim.cache.external
+ - shim.cache.node
+ - shim.cache.placeholder
+ - shim.cache.task
+ - shim.client
+ - shim.config
+ - shim.context
+ - shim.dispatcher
+ - shim.framework
+ - shim.fsm
+ - shim.predicates
+ - shim.resources
+ - shim.rmcallback
+ - shim.scheduler
+ - shim.scheduler.plugin
+ - shim.utils
diff --git a/versioned_docs/version-1.5.0/user_guide/sorting_policies.md b/versioned_docs/version-1.5.0/user_guide/sorting_policies.md
new file mode 100644
index 0000000..f5de40a
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/sorting_policies.md
@@ -0,0 +1,199 @@
+---
+id: sorting_policies
+title: Sorting Policies
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+The scheduler uses policies allow changing the scheduling behaviour without code changes.
+Policies can be set for:
+* [Applications](#application-sorting)
+* [Nodes](#node-sorting)
+* [Requests](#request-sorting)
+
+## Application sorting
+The application sorting policy is set for each queue via the config.
+A sorting policy setting is only effective on a _leaf_ queue.
+Each _leaf_ queue can use a different policy.
+
+A sorting policy only specifies the order in which the applications are sorted within a queue.
+That order is crucial in specifying which application is considered first when assigning resources.
+Sorting policies do _not_ affect the number of applications that are scheduled or active in the queue at the same time.
+All applications that have pending resource requests can and will be scheduled in a queue unless specifically filtered out.
+Even when applications are sorted using a first in first out policy multiple applications will run in a queue in parallel.
+
+A _parent_ queue will always use the fair policy to sort the child queues.
+
+The relative priority of child queues (in the case of _parent_ queue sorting)
+and applications (in the case of _leaf_ queue sorting) will be considered first.
+To ignore application and queue priorities when scheduling, set the queue
+property `application.sort.priority` to `disabled`.
+
+The following configuration entry sets the application sorting policy to `fifo` for the queue `root.sandbox`:
+```yaml
+partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: sandbox
+ properties:
+ application.sort.policy: fifo
+```
+
+The only applications that are considered during scheduling must have outstanding requests.
+A filter is applied _while_ sorting the applications to remove all that do not have outstanding requests.
+
+### FifoSortPolicy
+Short description: first in first out, based on application create time
+
+Config value: `fifo` (default)
+
+Before sorting, the applications are filtered and must have pending resource requests.
+
+After filtering the applications left are sorted based on the application create time stamp only, no other filtering is applied.
+Since applications can only be added while the system is locked there can never be two applications with the exact same time stamp.
+
+The result is that the oldest application that requests resources gets resources.
+Younger applications will be given resources when all the current requests of older applications have been fulfilled.
+
+### FairSortPolicy
+Short description: fair based on usage
+
+Config value: `fair`
+
+Before sorting the applications are filtered and must have pending resource requests.
+
+After filtering the applications left are sorted based on the application usage.
+The usage of the application is defined as all confirmed and unconfirmed allocations for the applications.
+All resources defined on the application will be taken into account when calculating the usage.
+
+The result is that the resources available are spread equally over all applications that request resources.
+
+### StateAwarePolicy
+Short description: limit of one (1) application in Starting or Accepted state
+
+Config value: `stateaware`
+
+**DEPRECATED:** The `stateaware` policy is **deprecated** in YuniKorn 1.5.0 and
+will be **removed** in YuniKorn 1.6.0. To preserve backwards compatibility,
+`stateaware` will become an alias for `fifo` in YuniKorn 1.6.0 and later.
+Users are encouraged to migrate to `fifo` and utilize either gang scheduling or
+`maxapplications` to limit concurrency instead.
+
+This sorting policy requires an understanding of the application states.
+Applications states are described in the [application states](developer_guide/scheduler_object_states.md#application-state) documentation.
+
+Before sorting applications the following filters are applied to all applications in the queue:
+The first filter is based on the application state.
+The following applications pass through the filter and generate the first intermediate list:
+* all applications in the state _running_
+* _one_ (1) application in the _starting_ state
+* if there are _no_ applications in the _starting_ state _one_ (1) application in the _accepted_ state is added
+
+The second filter takes the result of the first filter as an input.
+The preliminary list is filtered again: all applications _without_ a pending request are removed.
+
+After filtering based on status and pending requests the applications that remain are sorted.
+The final list is thus filtered twice with the remaining applications sorted on create time.
+
+To recap the _staring_ and _accepted_ state interactions:
+The application in the _accepted_ state is only added if there is no application in the _starting_ state.
+The application in the _starting_ state does not have to have pending requests.
+Any application in the _starting_ state will prevent _accepted_ applications from being added to the filtered list.
+
+For further details see the [Example run](design/state_aware_scheduling.md#example-run) in the design document.
+
+The result is that already running applications that request resources will get resources first.
+A drip feed of one new applications is added to the list of running applications to be allocated after all running applications.
+
+## Node sorting
+The node sorting policy is set for a partition via the config.
+Each partition can use a different policy.
+
+The following configuration entry sets the node sorting policy to `fair` for the partition `default`:
+```yaml
+partitions:
+ - name: default
+ nodesortpolicy:
+ type: fair
+```
+
+### FairnessPolicy
+Short description: available resource, descending order
+Config value: fair (default)
+Behaviour:
+Sort the list of nodes by the amount of available resources so that the node with the _highest_ amount of available resource is the first in the list.
+All resources defined on a node will be taken into account when calculating the usage.
+Resources of the same type are compared for the nodes.
+
+This results in a node with the lowest utilisation to be considered first for assigning new allocation.
+Resulting in a spread of allocations over all available nodes.
+Leading to an overall lower utilisation of the individual available nodes, unless the whole environment is highly utilised.
+Keeping the load on all nodes at a similar level does help
+In an environment that auto scales by adding new nodes this could trigger unexpected auto scale requests.
+
+### BinPackingPolicy
+Short description: available resource, ascending order
+Config value: binpacking
+Behaviour:
+Sort the list of nodes by the amount of available resources so that the node with the _lowest_ amount of available resource is the first in the list.
+All resources defined on a node will be taken into account when calculating the usage.
+Resources of the same type are compared for the nodes.
+
+This results in a node with the highest utilisation to be considered first for assigning new allocation.
+Resulting in a high(er) utilisation of a small(er) number of nodes, better suited for cloud deployments.
+
+## Resource weighting
+Node sorting policies may use the utilization of a node to determine ordering. Because nodes can have several unique
+resource types, a node's utilization is determined by a weighted average of its individual resource types. Resource
+weighting can be customized by using the `resourceweights` section of `nodesortpolicy`. If `resourceweights` is not
+present or empty, the default configuration sets the weight of both `vcore` and `memory` equally to `1.0`. All other
+resource types are ignored. Only resource types explicitly mentioned will have a weight.
+
+YuniKorn tracks CPU resources internally as the `vcore` resource type. This maps to the Kubernetes resource type `cpu`.
+All other resource types have consistent naming between YuniKorn and Kubernetes.
+
+For example, in the default configuration, if a node has `90%` of its CPU and `50%` of its memory allocated, the node
+will be considered to be `70%` utilized.
+
+The following configuration entry sets the weight of `vcore` to `4.0` and `memory` to `1.0` for the partition `default`.
+This will weight CPU usage four times higher than memory usage:
+```yaml
+partitions:
+ - name: default
+ nodesortpolicy:
+ type: fair
+ resourceweights:
+ vcore: 4.0
+ memory: 1.0
+```
+
+With this configuration, In this example, if a node has `90%` of its CPU and `50%` of its memory allocated, the node
+will be considered to be `82%` utilized.
+
+Note that weights are relative to each other, so specifying weights of `{ 4.0, 1.0 }` is equivalent to
+`{ 1.0, 0.25 }`. Negative weights are not allowed.
+
+## Request sorting
+There is currently one policy for sorting requests within an application.
+This policy is not configurable.
+Sorting requests is only possible based on the priority of the request.
+If there are multiple requests within an application that have the same priority the order of the requests is undetermined.
+This means that the order of requests with the same priority can, and most likely will, change between runs.
diff --git a/versioned_docs/version-1.5.0/user_guide/troubleshooting.md b/versioned_docs/version-1.5.0/user_guide/troubleshooting.md
new file mode 100644
index 0000000..05b5237
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/troubleshooting.md
@@ -0,0 +1,227 @@
+---
+id: troubleshooting
+title: Troubleshooting
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+## Scheduler logs
+
+### Understanding the linkage between Pod UUID, Task TaskID, AllocationAsk AllocationKey and Allocation AllocationID
+
+Pod is always submitted with `UID`, a unique identifier to differentiate various pods. When a pod is submitted, a `Task` gets created in the Shim. It uses `UID` of the POD as `TaskID` and passed as an `AllocationAsk` request to the core. `AllocationAsk` uses `Task`'s
+`TaskID` as `AllocationKey` and passed onto core for further processing. On receiving the ask request, Core tries to find a suitable
+`Allocation` using `AllocationAsk`'s `AllocationKey` as `AllocationID`. Understanding this flow and its linkage between different objects helps to debug the issues.
+
+An example has been described below to explain how pod's `UID` is getting translated with different name and passed through different objects.
+
+On task creation,
+
+```shell script
+2023-10-05T10:00:02.224Z INFO cache/context.go:832 task added {"appID": "yunikorn-dex-app-mqgh4dw2-autogen", "taskID": "849b762d-68c7-4cce-96e1-5acb545a8620", "taskState": "New"}
+```
+
+On processing allocation,
+
+```shell script
+2023-10-05T10:00:02.523Z INFO scheduler/partition.go:890 scheduler allocation processed {"appID": "yunikorn-dex-app-mqgh4dw2-autogen", "allocationKey": "849b762d-68c7-4cce-96e1-5acb545a8620", "allocationID": "849b762d-68c7-4cce-96e1-5acb545a8620-0", "allocatedResource": "map[memory:343932928 vcore:200]", "placeholder": false, "targetNode": "ip-10-130-86-3.eu-west-1.compute.internal"}
+```
+
+On binding the pod to node,
+
+```shell script
+2023-10-05T10:00:02.523Z INFO client/kubeclient.go:112 bind pod to node {"podName": "dbtdmchicdbdmchicdbstagingdata-1f8b6b53321f4cee9da13a7ac1f2a60c", "podUID": "849b762d-68c7-4cce-96e1-5acb545a8620", "nodeID": "ip-10-130-86-3.eu-west-1.compute.internal"}
+```
+
+### Retrieve scheduler logs
+
+Currently, the scheduler writes its logs to stdout/stderr, docker container handles the redirection of these logs to a
+local location on the underneath node, you can read more document [here](https://docs.docker.com/config/containers/logging/configure/).
+These logs can be retrieved by [kubectl logs](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#logs). Such as:
+
+```shell script
+// get the scheduler pod
+kubectl get pod -l component=yunikorn-scheduler -n yunikorn
+NAME READY STATUS RESTARTS AGE
+yunikorn-scheduler-766d7d6cdd-44b82 2/2 Running 0 33h
+
+// retrieve logs
+kubectl logs yunikorn-scheduler-766d7d6cdd-44b82 yunikorn-scheduler-k8s -n yunikorn
+```
+
+In most cases, this command cannot get all logs because the scheduler is rolling logs very fast. To retrieve more logs in
+the past, you will need to setup the [cluster level logging](https://kubernetes.io/docs/concepts/cluster-administration/logging/#cluster-level-logging-architectures).
+The recommended setup is to leverage [fluentd](https://www.fluentd.org/) to collect and persistent logs on an external storage, e.g s3.
+
+### Set Logging Level
+
+Edit the yunikorn-configs configmap:
+
+```shell script
+kubectl edit configmap/yunikorn-configs -n yunikorn
+```
+
+Add `log.level` to the `data` field of the configmap. For example setting `log.level` to `DEBUG` sets the logging
+level to `DEBUG`.
+
+```yaml
+apiVersion: v1
+data:
+ log.level: DEBUG
+ ...
+kind: ConfigMap
+metadata:
+ ...
+```
+
+The `log.level` value can be either numeric (-1 through 5) or textual (DEBUG through FATAL).
+
+| Value | Logging Level |
+|:------:|:-------------:|
+| -1 | DEBUG |
+| 0 | INFO |
+| 1 | WARN |
+| 2 | ERROR |
+| 3 | DPANIC |
+| 4 | PANIC |
+| 5 | FATAL |
+
+## Pods are stuck at Pending state
+
+If some pods are stuck at Pending state, that means the scheduler could not find a node to allocate the pod. There are
+several possibilities to cause this:
+
+### 1. Non of the nodes satisfy pod placement requirement
+
+A pod can be configured with some placement constraints, such as [node-selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector),
+[affinity/anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity),
+do not have certain toleration for node [taints](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/), etc.
+To debug such issues, you can describe the pod by:
+
+```shell script
+kubectl describe pod <pod-name> -n <namespace>
+```
+
+the pod events will contain the predicate failures and that explains why nodes are not qualified for allocation.
+
+### 2. The queue is running out of capacity
+
+If the queue is running out of capacity, pods will be pending for available queue resources. To check if a queue is still
+having enough capacity for the pending pods, there are several approaches:
+
+1) check the queue usage from yunikorn UI
+
+If you do not know how to access the UI, you can refer the document [here](../get_started/get_started.md#access-the-web-ui). Go
+to the `Queues` page, navigate to the queue where this job is submitted to. You will be able to see the available capacity
+left for the queue.
+
+2) check the pod events
+
+Run the `kubectl describe pod` to get the pod events. If you see some event like:
+`Application <appID> does not fit into <queuePath> queue`. That means the pod could not get allocated because the queue
+is running out of capacity.
+
+The pod will be allocated if some other pods in this queue is completed or removed. If the pod remains pending even
+the queue has capacity, that may because it is waiting for the cluster to scale up.
+
+## Obtain full state dump
+
+A Yunikorn state dump contains the every state object for every process which getting dumped. With endpoint to retrieve we can have many useful information in a single response for troubleshooting for example: list of partitions, list of applications which includes running, completed also historical application details, number of nodes, utilization of nodes, generic cluster information, cluster utilization details, container history and queues information.
+
+The state dump is a valuable resource that Yunikorn offers for use while troubleshooting.
+
+There are a few ways to obtain the full state dump.
+
+### 1. Scheduler URL
+
+STEPS:
+* Open the Scheduler URL in your browser window/tab and edit the URL as follows:
+* Replace `/#/dashboard` with `/ws/v1/fullstatedump`, (For example, `http://localhost:9889/ws/v1/fullstatedump`)
+* Press Enter
+
+That displays and provides an easy user experience to view live full state dump.
+
+### 2. Scheduler REST API
+
+With the below scheduler REST API returns information about full state dump used by the YuniKorn Scheduler.
+
+`curl -X 'GET' http://localhost:9889/ws/v1/fullstatedump -H 'accept: application/json'`
+
+For more details around the content of the state dump, please refer to the documentation on [retrieve-full-state-dump](api/scheduler.md#retrieve-full-state-dump)
+
+## Restart the scheduler
+
+:::note
+In accordance with best practices for troubleshooting, restarting the scheduler should only be done as a last effort to get everything back up and running. It should never be done before gathering all logs and state dumps.
+:::
+
+YuniKorn can recover its state upon a restart. YuniKorn scheduler pod is deployed as a deployment, restart the scheduler
+can be done by scale down and up the replica:
+
+```shell script
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=0
+kubectl scale deployment yunikorn-scheduler -n yunikorn --replicas=1
+```
+
+## Gang Scheduling
+
+### 1. No placeholders created, app's pods are pending
+
+*Reason*: This is usually because the app is rejected by the scheduler, therefore non of the pods are scheduled.
+The common reasons caused the rejection are: 1) The taskGroups definition is invalid. The scheduler does the
+sanity check upon app submission, to ensure all the taskGroups are defined correctly, if these info are malformed,
+the scheduler rejects the app; 2) The total min resources defined in the taskGroups is bigger than the queues' max
+capacity, scheduler rejects the app because it won't fit into the queue's capacity. Check the pod event for relevant messages,
+and you will also be able to find more detail error messages from the schedulers' log.
+
+*Solution*: Correct the taskGroups definition and retry submitting the app.
+
+### 2. Not all placeholders can be allocated
+
+*Reason*: The placeholders also consume resources, if not all of them can be allocated, that usually means either the queue
+or the cluster has no sufficient resources for them. In this case, the placeholders will be cleaned up after a certain
+amount of time, defined by the `placeholderTimeoutInSeconds` scheduling policy parameter.
+
+*Solution*: Note, if the placeholder timeout reaches, currently the app will transit to failed state and can not be scheduled
+anymore. You can increase the placeholder timeout value if you are willing to wait for a longer time. In the future, a fallback policy
+might be added to provide some retry other than failing the app.
+
+### 3. Not all placeholders are swapped
+
+*Reason*: This usually means the actual app's pods are less than the minMembers defined in the taskGroups.
+
+*Solution*: Check the `minMember` in the taskGroup field and ensure it is correctly set. The `minMember` can be less than
+the actual pods, setting it to bigger than the actual number of pods is invalid.
+
+### 4.Placeholders are not cleaned up when the app terminated
+
+*Reason*: All the placeholders are set an [ownerReference](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#owners-and-dependents)
+to the first real pod of the app, or the controller reference. If the placeholder could not be cleaned up, that means
+the garbage collection is not working properly.
+
+*Solution*: check the placeholder `ownerReference` and the garbage collector in Kubernetes.
+
+
+## Still got questions?
+
+No problem! The Apache YuniKorn community will be happy to help. You can reach out to the community with the following options:
+
+1. Post your questions to dev@yunikorn.apache.org
+2. Join the [YuniKorn slack channel](https://join.slack.com/t/yunikornworkspace/shared_invite/enQtNzAzMjY0OTI4MjYzLTBmMDdkYTAwNDMwNTE3NWVjZWE1OTczMWE4NDI2Yzg3MmEyZjUyYTZlMDE5M2U4ZjZhNmYyNGFmYjY4ZGYyMGE) and post your questions to the `#yunikorn-user` channel.
+3. Join the [community sync up meetings](http://yunikorn.apache.org/community/get_involved#community-meetings) and directly talk to the community members.
diff --git a/versioned_docs/version-1.5.0/user_guide/use_case.md b/versioned_docs/version-1.5.0/user_guide/use_case.md
new file mode 100644
index 0000000..99e47e1
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/use_case.md
@@ -0,0 +1,483 @@
+---
+id: use_cases
+title: Use Cases
+---
+
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ -->
+
+## Introduction
+
+Yunikorn offers a range of features, including advanced capabilities like hierarchical resource queues, access control lists, resource limits, preemption, priority, and placement rules for managing your cluster. This page presents a real-world scenario to demonstrate the practical application of these features.
+
+We will now introduce the various functions and configurations of Yunikorn in sequence.
+
+The following will be included in this article：
+
+- Access control with ACL
+- Placement of different users
+- Limit usable resources on a queue level
+- Preemption and priority scheduling with fencing
+
+---
+
+## Prerequisite
+
+Before configuring yunikorn-config, we need to create users using [Authentication](https://kubernetes.io/docs/reference/access-authn-authz/authentication/) and [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) from Kubernetes.
+
+To create the necessary users for the example, you can download the `create_user.sh` script directly from yunikorn-k8shim under [deployment/examples/authz/k8s-api-access](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/examples/authz/k8s-api-access).
+
+Here are the users we need to create:
+
+| user | group |
+|-----------|-----------|
+| admin | admin |
+| sue | group-a |
+| bob | group-a |
+| kim | group-b |
+| yono | group-b |
+| anonymous | anonymous |
+
+After the user is created, the pod can be obtained by the following command to confirm the creation is successful：
+
+```yaml
+kubectl --context=sue-context get pod
+```
+
+When you are done testing, you can run `./remove-user.sh` in the folder to delete all users.
+
+---
+
+## Partition and Queue Configuration
+
+In this use case, we configure the cluster for two purposes - system management and multi-tenancy, with two groups of tenants. Below is the queue configuration:
+
+- Root
+ - system
+ - tenants
+ - group-a
+ - group-b
+
+```yaml
+kind: ConfigMap
+metadata:
+ name: yunikorn-configs
+ namespace: yunikorn
+apiVersion: v1
+data:
+ queues.yaml: |
+ partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: system
+ - name: tenants
+ queues:
+ - name: group-a
+ - name: group-b
+```
+
+See the documentation on [Partition and Queue Configuration](queue_config) for more information.
+
+---
+
+## User/Group Resolution & ACL
+
+In a multi-tenant environment, we want each user's workload to be independent from one another, with tenants restricted to using specific queues.
+
+In Yunikorn, there are two steps to manage users:
+
+1. The Admission Controller resolves the username and group and adds the user's informantion to the Annotation.
+2. Yunikorn compares whether the user is in the ACL of a specific queue to determine whether the user has permission to deploy applications on that queue.
+
+### Explanation of Configuration
+
+In the following example, we configure the configuration based on the following:
+
+| user | group | Queue allowed to put |
+|-----------|-----------|----------------------|
+| admin | admin | system |
+| sue | group-a | root.tenants.group-a |
+| bob | group-a | root.tenants.group-a |
+| kim | group-b | root.tenants.group-b |
+| yono | group-b | root.tenants.group-b |
+| anonymous | anonymous | x |
+
+In this configuration, we allow users in the admin group to use the system queue by setting `adminacl: admin` for the `root.system` queue. We also allow the group-a and group-b groups to use their respective queues (`root.tenants.group-a` and `root.tenants.group-b`) by setting `adminacl: group-a` and `adminacl: group-b` for each queue, respectively.
+
+Configuration for testing :
+
+```yaml
+kind: ConfigMap
+metadata:
+ name: yunikorn-configs
+ namespace: yunikorn
+apiVersion: v1
+data:
+ admissionController.accessControl.externalGroups: "admin,group-a,group-b"
+ queues.yaml: |
+ partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: system
+ adminacl: " admin"
+ - name: tenants
+ queues:
+ - name: group-a
+ adminacl: " group-a"
+ - name: group-b
+ adminacl: " group-b"
+```
+
+### Testing
+
+In this test scenario, we utilize three users to create a Pod, but the access control list restricts each user to create Pods only in the allowed queue.
+
+Among these users, one is named Sue and belongs to the group-a. Whenever Sue tries to create a Pod, the admission controller will first add the user's name and group to the Pod's annotation:
+
+```yaml
+Annotations:
+ ...
+ yunikorn.apache.org/user.info: {"user":"sue","groups":["group-a","system:authenticated"]}
+ ...
+```
+
+The scheduler will then determine whether to allow or block the user's pod based on the access control list of the assigned queue.
+
+Here are the results for different users assigned to different queues. You can use the [YAML file](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/examples/authz/acl) we provide to test :
+
+| user, group | Assign queue | result | YAML filename |
+|----------------------|----------------------|---------|---------------|
+| sue, group-a | root.tenants.group-a | created | nginx-1.yaml |
+| sue, group-a | root.tenants.group-b | blocked | nginx-1.yaml |
+| kim, group-b | root.tenants.group-a | blocked | nginx-2.yaml |
+| kim, group-b | root.tenants.group-b | created | nginx-2.yaml |
+| anonymous, anonymous | root.tenants.group-a | blocked | nginx-3.yaml |
+| anonymous, anonymous | root.tenants.group-b | blocked | nginx-3.yaml |
+
+See the documentation on [User & Group Resolution](usergroup_resolution) or [ACLs](acls) for more information.
+
+---
+
+## Placement of different users
+
+To simplify the user's experience, the scheduler can dynamically assign the application to a queue and create a new queue if necessary. By setting up the placement rules properly, the user won't need to specify the queue to run their application on.
+
+### Explanation of Configuration
+
+Yunikorn offers extensive support for placement rules, allowing users to assign queues based on the namespace or username of the Pod, or to create queues based on user specifications.
+
+Let's take the "admin" user as an example to explain the configuration. We want the scheduler to create a new queue based on the queue provided by the "admin":
+
+```yaml
+placementrules:
+ - name: provided
+ create: true
+```
+
+To further restrict users and provide different placement rules for different users, we need to use `filter` since the previous configuration cannot achieve this.
+
+Additionally, we want to separate the system and tenant queues, and for queues created by the admin, we want them to be under the system queue. To do so, we use the `parent` attribute.
+
+To allow the queue to be designated as a parent, we need to set `parent: true` in the parent queue.
+
+Finally, we extend the configuration as follows:
+
+```yaml
+placementrules:
+ - name: provided
+ create: true
+ parent: # Specify the parent of the created queue as root.system
+ name: fixed
+ value: root.system
+ filter: # Only admin is allowed to use the rule
+ type: allow
+ users:
+ - admin
+ groups:
+ - admin
+```
+
+In the following example, we configure the configuration based on the following:
+
+| group | placement rule | fixed parent |
+|---------|----------------|----------------------|
+| admin | provided | root.system |
+| group-a | username | root.tenants.group-a |
+| group-b | namespace | root.tenants.group-b |
+
+Configuration for testing :
+
+```yaml
+kind: ConfigMap
+metadata:
+ name: yunikorn-configs
+ namespace: yunikorn
+apiVersion: v1
+data:
+ admissionController.accessControl.externalGroups: "admin,group-a,group-b"
+ queues.yaml: |
+ partitions:
+ - name: default
+ placementrules:
+ - name: provided
+ create: true
+ parent:
+ name: fixed
+ value: root.system
+ filter:
+ type: allow
+ users:
+ - admin
+ groups:
+ - admin
+ - name: user
+ create: true
+ filter:
+ type: allow
+ groups:
+ - group-a
+ parent:
+ name: fixed
+ value: root.tenants.group-a
+ - name: tag
+ value: namespace
+ create: true
+ filter:
+ type: allow
+ groups:
+ - group-b
+ parent:
+ name: fixed
+ value: root.tenants.group-b
+ queues:
+ - name: root
+ queues:
+ - name: system
+ adminacl: " admin"
+ parent: true # Let the queue be designated as the parent queue
+ - name: tenants
+ parent: true
+ queues:
+ - name: group-a
+ adminacl: " group-a"
+ parent: true
+ - name: group-b
+ adminacl: " group-b"
+ parent: true
+```
+
+### Testing
+
+In this test example, we use three users to verify all the placement rules.
+
+The following results are generated when creating a Pod according to different rules. You can use the [YAML file](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/examples/authz/placement-rules) we provide for testing:
+
+| placement rule | user, group | provide queue | namespace | Expected to be placed on | YAML filename |
+|------------------------|--------------|---------------------------|-----------|---------------------------|------------------|
+| provided | admin, admin | root.system.high-priority | | root.system.high-priority | nginx-admin.yaml |
+| provided | admin, admin | root.system.low-priority | | root.system.low-priority | nginx-admin.yaml |
+| username | sue, group-a | | | root.tenants.group-a.sue | nginx-sue.yaml |
+| tag (value: namespace) | kim, group-b | | dev | root.tenants.group-b.dev | nginx-kim.yaml |
+| tag (value: namespace) | kim, group-b | | test | root.tenants.group-b.test | nginx-kim.yaml |
+
+See the documentation on [App Placement Rules](placement_rules) for more information.
+
+---
+
+## Limit usable resources on a queue level
+
+To avoid unfair resource usage, we can limit and reserve the amount of resources per queue.
+
+### Explanation of Configuration
+
+In the following example, we configure the configuration based on the following:
+
+| queue | guaranteed | | max | |
+|----------------------|------------|-------|-----------|-------|
+| | memory(G) | vcore | memory(G) | vcore |
+| root | x | x | 15.6 | 16 |
+| root.system | 2 | 2 | 6 | 6 |
+| root.tenants | 2 | 2 | 4 | 8 |
+| root.tenants.group-a | 1 | 1 | 2 | 4 |
+| root.tenants.group-b | 1 | 1 | 2 | 4 |
+
+Configuration for testing :
+
+```yaml
+kind: ConfigMap
+metadata:
+ name: yunikorn-configs
+ namespace: yunikorn
+apiVersion: v1
+data:
+ admissionController.accessControl.externalGroups: "admin,^group-$"
+ queues.yaml: |
+ partitions:
+ - name: default
+ queues:
+ - name: root
+ queues:
+ - name: system
+ adminacl: " admin"
+ resources:
+ guaranteed:
+ {memory: 2G, vcore: 2}
+ max:
+ {memory: 6G, vcore: 6}
+ - name: tenants
+ resources:
+ guaranteed:
+ {memory: 2G, vcore: 2}
+ max:
+ {memory: 4G, vcore: 8}
+ queues:
+ - name: group-a
+ adminacl: " group-a"
+ resources:
+ guaranteed:
+ {memory: 1G, vcore: 1}
+ max:
+ {memory: 2G, vcore: 4}
+ - name: group-b
+ adminacl: " group-b"
+ resources:
+ guaranteed:
+ {memory: 1G, vcore: 1}
+ max:
+ {memory: 2G, vcore: 4}
+```
+
+### Testing
+
+In the following example, we restrict `root.tenants.group-a` to use a maximum of `{memory: 2G, vcore: 4}` resources.
+
+When group-A is deployed in `root.tenants.group-a` and the required resources exceed the limit, the remaining pods will be blocked.
+
+The results of deploying Pods in different queues are shown below. You can use the [YAML file](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/examples/authz/resource-limits) we provide for testing.
+
+| user, group | Resource Limits for Destination Queues | request resources for each replicas | replica | result | YAML filename |
+|--------------|----------------------------------------|-------------------------------------|---------|----------------------------------------------------------|------------------|
+| admin, admin | {memory: 6G, vcore: 6} | {memory: 512M, vcore: 250m} | 1 | run all replica | nginx-admin.yaml |
+| sue, group-A | {memory: 2G, vcore: 4} | {memory: 512M, vcore: 500m} | 5 | run 3 replica (4 replica will exceed the resource limit) | nginx-sue.yaml |
+
+See the documentation on [Partition and Queue Configuration #Resources](queue_config#resources) for more information.
+
+---
+
+## Preemption & Priority scheduling with fencing
+
+YuniKorn supports priority scheduling, where priorities can be assigned to each task and also to each queue.
+
+This section demonstrates how to configure priority in a queue. If you want to configure the priority of each task, you can learn more about it from document Pod Priority and Preemption on Kubernete .
+
+### Explanation of Configuration
+
+In the following example, we configure the configuration based on the following:
+
+| queue | offset |
+|-----------------------------|-------------|
+| root | |
+| root.system | |
+| root.system.high-priority | 1000 |
+| root.system.normal-priority | 0 |
+| root.system.low-priority | -1000 |
+| root.tenants | 0 (fenced) |
+| root.tenants.group-a | 20 (fenced) |
+| root.tenants.group-b | 5 (fenced) |
+
+By default, all priorities are globally scoped, which means that all high-priority queues will be served first. However, we can also limit the priority to certain queues.
+
+The following configuration sets up a fence to ensure that the priorities of the `root.tenants` queues (and their sub-queues) are evaluated internally.
+
+```yaml
+kind: ConfigMap
+metadata:
+ name: yunikorn-configs
+ namespace: yunikorn
+apiVersion: v1
+data:
+ queues.yaml: |
+ partitions:
+ - name: default
+ queues:
+ - name: root
+ properties:
+ application.sort.policy: fifo # default value: fifo
+ application.sort.priority: enabled # default value: enable
+ queues:
+ - name: system
+ adminacl: " admin"
+ queues:
+ - name: high-priority
+ properties:
+ priority.offset: "1000"
+ - name: normal-priority
+ properties:
+ priority.offset: "0"
+ - name: low-priority
+ properties:
+ priority.offset: "-1000"
+ - name: tenants
+ properties:
+ priority.policy: "fence"
+ queues:
+ - name: group-a
+ adminacl: " group-a"
+ properties:
+ priority.offset: "20"
+ - name: group-b
+ adminacl: " group-b"
+ properties:
+ priority.offset: "5"
+```
+
+### Testing
+
+**Case 1: priority**
+
+In the first test, we deploy an equal number of Pods with identical resource requests to three queues with different priorities. Without any priorities, we would expect to see an equal number of Pods deployed to each queue.
+
+However, with priorities and limited resources, the high-priority queue can deploy all of its Pods, the medium-priority queue can deploy some Pods, and the low-priority queue won't be able to deploy any Pods until resources are released by the high-priority queue.
+
+In the following tests, we run the environment with a node resource limit of `{memory:16GB, vcore:16}`. Note that results will vary based on the environment, and you can modify the [YAML file](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/examples/authz/priority) we provide to achieve similar results.
+
+| queue | offset | # of deploy apps | # of apps accept by yunikorn | YAML filename |
+|-----------------------------|--------|------------------|------------------------------|---------------|
+| root.system.low-priority | 1000 | 8 | 8 | system.yaml |
+| root.system.normal-priority | 0 | 8 | 5 | system.yaml |
+| root.system.high-priority | -1000 | 8 | 0 | system.yaml |
+
+**Case 2: priority-fenced**
+
+In the second test, we deploy the same number of Pods with the same resource requests to three different priority queues. However, this time, two queues are configured with a fence.
+
+While scheduling the task. Even though `root.tenants.group-a` has a higher priority than the other two queues, the scheduler will still execute root.system.normal first, which is in the global scope. Then, the scheduler will compare priorities within the local scope of `root.tenants`.
+
+For the following tests, we run them in an environment with node resources of `{memory:16GB, vcore:16}`. The results will vary in different environments, but you can obtain similar results by modifying the [YAML file](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/examples/authz/priority) we provide.
+
+| queue | offset | # of deploy apps | # of apps accept by yunikorn | YAML filename |
+|-----------------------------|-------------|------------------|------------------------------|------------------|
+| root.system.normal-priority | 0 (global) | 7 | 7 | nginx-admin.yaml |
+| root.tenants.group-a | 20 (fenced) | 7 | 6 | nginx-sue.yaml |
+| root.tenants.group-b | 5 (fenced) | 7 | 0 | nginx-kim.yaml |
+
+See the documentation on [App & Queue Priorities](priorities) for more information.
diff --git a/versioned_docs/version-1.5.0/user_guide/usergroup_resolution.md b/versioned_docs/version-1.5.0/user_guide/usergroup_resolution.md
new file mode 100644
index 0000000..3ecde06
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/usergroup_resolution.md
@@ -0,0 +1,120 @@
+---
+id: usergroup_resolution
+title: User & Group Resolution
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## User resolution
+
+User information is an important aspect of the scheduling cycle. It is one of the key identifier that can be used to determine the queue to which a job should be submitted. The Yunikorn Scheduler relies on the K8s Shim to provide user information. In the world of Kubernetes, there is no object defined that identfies the actual user. This is by design and more information can be found [here](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#users-in-kubernetes).
+
+In Yunikorn, there are two ways of handling users and groups. The first is the legacy way, which uses the label `yunikorn.apache.org/username`. If this label is set on a pod, then the value is automatically extracted in the shim and will be used accordingly. Group resolution is also done in the shim and is disabled by default. The problem with this approach is twofold: user restrictions can be easily bypassed because the submitter is free to set this label to any value, therefore this only be used in a trusted environment. The second is that identifying the groups in the shim is too late: users can be assigned to multiple groups, but depending on the authentication mechanism (X509, tokens, LDAP, etc) it might be very difficult to look up which group a user belongs to. Since these limitations are significant, this method is kept for backward compatibility reasons and will likely be removed in the future.
+
+A more reliable and robust mechanism is using the `yunikorn.apache.org/user.info` annotation, where the user information can be set externally by an allowed list of users or groups or the admission controller can attach this automatically to every workload.
+
+## Legacy user handling
+
+### Using the `yunikorn.apache.org/username` label
+
+Since, Kubernetes has no pre-defined field or resource for user information and individual cluster deployments with unique user identification tools can vary, we have defined a standard way of identifying the user. Yunikorn requires a Kubernetes [Label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) added. Using the [recommendation](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/) provided here, the default label is defined as below:
+
+| Label | Value |
+|--------------------------------|--------------------------------------------------------------------------------------------------------------|
+| yunikorn.apache.org/username | User name. It can have duplicate entries but only the first value will be used. The default user is `nobody` |
+
+Example:
+```yaml
+metadata:
+ labels:
+ yunikorn.apache.org/username: "john"
+```
+:::tip
+In order to make this field uniquiely identifiable to the authorized user, the suggestion is to add this label as an immutable field by the user identification tool used by the cluster administrators. The cluster administrators or users are free to use any method or tool to add this field and value. This includes adding it manually at the time of submission.
+:::
+
+:::note Assumption
+Assumption:
+ Yunikorn assumes that all pods belonging to an application are owned by the same user. We recommend that the user label is added to every pod of an app. This is to ensure that there is no discrepency.
+:::
+
+The `yunikorn.apache.org/username` key can be customized by overriding the default value using the `USER_LABEL_KEY`env variable in the [K8s Deployment](https://github.com/apache/yunikorn-release/blob/master/helm-charts/yunikorn/templates/deployment.yaml). This is particularly useful in scenarios where the user label is already being added or if the label has to be modified for some secuirty reasons.
+
+```yaml
+ env:
+ - name: USER_LABEL_KEY
+ value: "custom_user_label"
+```
+
+### Group resolution
+
+Group membership resolution is pluggables and is defined here. Groups do not have to be part of provided user and group object. When the object is added to the cache the groups are automatically resolved based on the resolution that is configured.
+The resolver which is linked to the cache can be set per partition.
+
+The default group resolver is "no resolver".
+This resolver just echos the user name and a primary group with the same name as the user.
+
+Other resolvers are:
+* OS resolver
+* test resolver
+
+## The new, recommended way of handling users
+
+Since Yunikorn 1.2 a more sophisticated way of user/group resolution is available.
+
+In this mode, Yunikorn no longer relies on the `yunikorn.apache.org/username` label, instead, the annotation `yunikorn.apache.org/user.info` is attached to the workload. The value is simple JSON, which defines the user name and groups:
+
+```yaml
+metadata:
+ annotations:
+ yunikorn.apache.org/user.info: "
+ {
+ \"user\": \"yunikorn\",
+ \"groups\": [
+ \"developers\",
+ \"devops\"
+ ]
+ }"
+```
+
+However, to enhance security, the following is enforced in the admission controller:
+* not every user in the cluster is allowed to attach this annotation, only those which are configured
+* if the annotation is missing, the admission controller will add this information automatically
+* attempts to change this annotation will be rejected
+
+We also no longer do this on pods only, but also on Deployments, ReplicaSets, DeamonSets, StatefulSets, Jobs and CronJobs.
+
+Group resolution is no longer necessary inside the shim.
+
+### Configuring the admission controller
+
+The admission controller can be configured with the `yunikorn-configs` configmap. All entries start with the prefix `admissionController.accessControl.`.
+
+| Variable | Default value | Description |
+|--------------------|---------------------------------------|----------------------------------------------------------------------------|
+| `bypassAuth` | false | Allow any external user to create pods with user information set |
+| `trustControllers` | true | Allow Kubernetes controller users to create pods with user information set |
+| `systemUsers` | "^system:serviceaccount:kube-system:" | Regular expression for the allowed controller service account list |
+| `externalUsers` | "" | Regular expression for the allowed external user list |
+| `externalGroups` | "" | Regular expression for the allowed external group list |
+
+If `bypassAuth` is set to true the admission controller will not add the annotation to a pod if the annotation is not present and the deprecated user labell is set. If the annotation is not set and the user label is not set the new annotation will be added. In the case that `bypassAuth` is false, the default, the admission controller will always add the new annotation, regardless of the existence of the deprecated label.
+
+In certain scenarios, users and groups must be provided to Yunikorn upon submission because the user and group management is provided by external systems and the lookup mechanism is not trivial. In these cases, the `externalUsers` and `externalGroups` can be configured which are treated as regular expressions. Matching users and groups are allowed to set the `yunikorn.apache.org/user.info` annotation to any arbitrary value. Since this has implications which affects scheduling inside Yunikorn, these properties must be set carefully.
diff --git a/versioned_docs/version-1.5.0/user_guide/workloads/run_flink.md b/versioned_docs/version-1.5.0/user_guide/workloads/run_flink.md
new file mode 100644
index 0000000..bb90027
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/workloads/run_flink.md
@@ -0,0 +1,66 @@
+---
+id: run_flink
+title: Run Flink Jobs
+description: How to run Flink jobs with YuniKorn
+image: https://svn.apache.org/repos/asf/flink/site/img/logo/png/100/flink_squirrel_100_color.png
+keywords:
+ - spark
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+It's very easy to run [Apache Flink](https://flink.apache.org/) on Kubernetes with YuniKorn. Depending on which mode is
+used to run Flink on Kubernetes, the configuration is slight different.
+
+## Standalone mode
+
+Please follow [Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/kubernetes.html) to get details and examples of standalone deploy mode.
+In this mode, we can directly add required labels (applicationId and queue) in Deployment/Job spec to run flink application with YuniKorn scheduler, see the [overview](./workload_overview.md) for the label specification.
+
+## Native mode
+
+Please follow [Native Kubernetes Setup](https://ci.apache.org/projects/flink/flink-docs-stable/ops/deployment/native_kubernetes.html) to get details and examples of native deploy mode.
+Running flink application with YuniKorn scheduler in native mode is only supported for flink 1.11 or above, we can leverage two flink configurations `kubernetes.jobmanager.labels` and `kubernetes.taskmanager.labels` to set the required labels.
+Examples:
+
+* Start a flink session
+```
+./bin/kubernetes-session.sh \
+ -Dkubernetes.cluster-id=<ClusterId> \
+ -Dtaskmanager.memory.process.size=4096m \
+ -Dkubernetes.taskmanager.cpu=2 \
+ -Dtaskmanager.numberOfTaskSlots=4 \
+ -Dresourcemanager.taskmanager-timeout=3600000 \
+ -Dkubernetes.jobmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ -Dkubernetes.taskmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox
+```
+
+* Start a flink application
+```
+./bin/flink run-application -p 8 -t kubernetes-application \
+ -Dkubernetes.cluster-id=<ClusterId> \
+ -Dtaskmanager.memory.process.size=4096m \
+ -Dkubernetes.taskmanager.cpu=2 \
+ -Dtaskmanager.numberOfTaskSlots=4 \
+ -Dkubernetes.container.image=<CustomImageName> \
+ -Dkubernetes.jobmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ -Dkubernetes.taskmanager.labels=applicationId:MyOwnApplicationId,queue:root.sandbox \
+ local:///opt/flink/usrlib/my-flink-job.jar
+```
\ No newline at end of file
diff --git a/versioned_docs/version-1.5.0/user_guide/workloads/run_mpi.md b/versioned_docs/version-1.5.0/user_guide/workloads/run_mpi.md
new file mode 100644
index 0000000..f69f8c7
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/workloads/run_mpi.md
@@ -0,0 +1,112 @@
+---
+id: run_mpi
+title: Run MPI Jobs
+description: How to run MPI jobs with YuniKorn
+keywords:
+ - mpi
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This guide walks through how to setup the [MPI Operator](https://github.com/kubeflow/mpi-operator) and to how to run a MPIJob with the YuniKorn scheduler.
+
+## Installing the MPI Operator
+
+You can use the following command to install the mpi operator. If you have problems with installation,
+please refer to [this doc](https://github.com/kubeflow/mpi-operator) for details.
+```
+kubectl create -f https://raw.githubusercontent.com/kubeflow/mpi-operator/master/deploy/v2beta1/mpi-operator.yaml
+```
+
+## Run a MPI Job
+
+This example shows to run a pure MPI application.
+
+The program prints some basic information about the workers.
+Then, it calculates an approximate value for pi.
+
+Here is a Pi YAML example [example](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/examples/mpioperator/Pi/pi.yaml).
+```yaml
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+ name: pi
+spec:
+ slotsPerWorker: 1
+ runPolicy:
+ cleanPodPolicy: Running
+ ttlSecondsAfterFinished: 60
+ sshAuthMountPath: /home/mpiuser/.ssh
+ mpiReplicaSpecs:
+ Launcher:
+ replicas: 1
+ template:
+ labels:
+ applicationId: "mpi_job_pi"
+ queue: root.mpi
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - image: mpioperator/mpi-pi
+ name: mpi-launcher
+ securityContext:
+ runAsUser: 1000
+ command:
+ - mpirun
+ args:
+ - -n
+ - "2"
+ - /home/mpiuser/pi
+ resources:
+ limits:
+ cpu: 1
+ memory: 1Gi
+ Worker:
+ replicas: 2
+ template:
+ labels:
+ applicationId: "mpi_job_pi"
+ queue: root.mpi
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - image: mpioperator/mpi-pi
+ name: mpi-worker
+ securityContext:
+ runAsUser: 1000
+ command:
+ - /usr/sbin/sshd
+ args:
+ - -De
+ - -f
+ - /home/mpiuser/.sshd_config
+ resources:
+ limits:
+ cpu: 1
+ memory: 1Gi
+```
+Create the MPIJob.
+```
+kubectl create -f deployments/examples/mpioperator/Pi/pi.yaml
+```
+
+We added Yunikorn labels to the Pi example to demonstrate using the yunikorn scheduler.
+
+
diff --git a/versioned_docs/version-1.5.0/user_guide/workloads/run_nvidia.md b/versioned_docs/version-1.5.0/user_guide/workloads/run_nvidia.md
new file mode 100644
index 0000000..b756488
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/workloads/run_nvidia.md
@@ -0,0 +1,353 @@
+---
+id: run_nvidia
+title: Run NVIDIA GPU Jobs
+description: How to run generic example of GPU scheduling with Yunikorn.
+keywords:
+ - NVIDIA GPU
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## Yunikorn with NVIDIA GPUs
+This guide gives an overview of how to set up NVIDIA Device Plugin which enable user to run GPUs with Yunikorn, for more details please check [**NVIDIA device plugin for Kubernetes**](https://github.com/NVIDIA/k8s-device-plugin#nvidia-device-plugin-for-kubernetes).
+
+### Prerequisite
+Before following the steps below, Yunikorn need to deploy on the Kubernetes with GPUs.
+
+### Install NVIDIA Device Plugin
+Add the nvidia-device-plugin helm repository.
+```
+helm repo add nvdp https://nvidia.github.io/k8s-device-plugin
+helm repo update
+helm repo list
+```
+
+Verify the latest release version of the plugin is available.
+```
+helm search repo nvdp --devel
+NAME CHART VERSION APP VERSION DESCRIPTION
+nvdp/nvidia-device-plugin 0.14.1 0.14.1 A Helm chart for ...
+```
+
+Deploy the device plugin
+```
+kubectl create namespace nvidia
+helm install nvidia-device-plugin nvdp/nvidia-device-plugin \
+ --namespace nvidia \
+ --create-namespace \
+ --version 0.14.1
+```
+
+Check the status of the pods to ensure NVIDIA device plugin is running
+```
+kubectl get pods -A
+
+NAMESPACE NAME READY STATUS RESTARTS AGE
+kube-flannel kube-flannel-ds-j24fx 1/1 Running 1 (11h ago) 11h
+kube-system coredns-78fcd69978-2x9l8 1/1 Running 1 (11h ago) 11h
+kube-system coredns-78fcd69978-gszrw 1/1 Running 1 (11h ago) 11h
+kube-system etcd-katlantyss-nzxt 1/1 Running 3 (11h ago) 11h
+kube-system kube-apiserver-katlantyss-nzxt 1/1 Running 4 (11h ago) 11h
+kube-system kube-controller-manager-katlantyss-nzxt 1/1 Running 3 (11h ago) 11h
+kube-system kube-proxy-4wz7r 1/1 Running 1 (11h ago) 11h
+kube-system kube-scheduler-katlantyss-nzxt 1/1 Running 4 (11h ago) 11h
+nvidia nvidia-device-plugin-1659451060-c92sb 1/1 Running 1 (11h ago) 11h
+```
+
+### Testing NVIDIA Device Plugin
+Create a gpu test yaml file.
+```yaml
+# gpu-pod.yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ name: gpu-pod
+spec:
+ restartPolicy: Never
+ containers:
+ - name: cuda-container
+ image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda11.7.1-ubi8
+ resources:
+ limits:
+ nvidia.com/gpu: 1 #requesting 1 GPU
+ tolerations:
+ - key: nvidia.com/gpu
+ operator: Exists
+ effect: NoSchedule
+```
+Deploy the application.
+```
+kubectl apply -f gpu-pod.yaml
+```
+Check the logs to ensure the app completed successfully.
+```
+kubectl get pod gpu-pod
+
+NAME READY STATUS RESTARTS AGE
+gpu-pod 0/1 Completed 0 9d
+```
+Check the result.
+```
+kubectl logs gpu-pod
+
+[Vector addition of 50000 elements]
+Copy input data from the host memory to the CUDA device
+CUDA kernel launch with 196 blocks of 256 threads
+Copy output data from the CUDA device to the host memory
+Test PASSED
+Done
+```
+
+---
+## Enable GPU Time-Slicing (Optional)
+GPU time-slicing allow multi-tenant to share single GPU.
+To know how the GPU time-slicing works, please refer to [**Time-Slicing GPUs in Kubernetes**](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/gpu-sharing.html#introduction). This page covers ways to enable GPU scheduling in Yunikorn using [**NVIDIA GPU Operator**](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/gpu-operator).
+
+
+### Configuration
+Specify multiple configurations in a `ConfigMap` as in the following example.
+```yaml
+# time-slicing-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: time-slicing-config
+ namespace: nvidia
+data:
+ a100-40gb: |-
+ version: v1
+ sharing:
+ timeSlicing:
+ resources:
+ - name: nvidia.com/gpu
+ replicas: 8
+ - name: nvidia.com/mig-1g.5gb
+ replicas: 2
+ - name: nvidia.com/mig-2g.10gb
+ replicas: 2
+ - name: nvidia.com/mig-3g.20gb
+ replicas: 3
+ - name: nvidia.com/mig-7g.40gb
+ replicas: 7
+ rtx-3070: |-
+ version: v1
+ sharing:
+ timeSlicing:
+ resources:
+ - name: nvidia.com/gpu
+ replicas: 8
+```
+
+:::note
+If the GPU type in nodes do not include the a100-40gb or rtx-3070, you could modify the yaml file based on existing GPU types.
+For example, there are only multiple rtx-2080ti in the local kubernetes cluster.
+MIG is not supported by rtx-2080ti, so it could not replace the a100-40gb.
+Time slicing is supported by rtx-2080ti, so it could replace rtx-3070.
+:::
+
+:::info
+MIG support was added to Kubernetes in 2020. Refer to [**Supporting MIG in Kubernetes**](https://www.google.com/url?q=https://docs.google.com/document/d/1mdgMQ8g7WmaI_XVVRrCvHPFPOMCm5LQD5JefgAh6N8g/edit&sa=D&source=editors&ust=1655578433019961&usg=AOvVaw1F-OezvM-Svwr1lLsdQmu3) for details on how this works.
+:::
+
+Create a `ConfigMap` in the operator namespace.
+```bash
+kubectl create namespace nvidia
+kubectl create -f time-slicing-config.yaml
+```
+
+### Install NVIDIA GPU Operator
+Add the nvidia-gpu-operator helm repository.
+```bash
+helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
+helm repo update
+helm repo list
+```
+
+Enabling shared access to GPUs with the NVIDIA GPU Operator.
+- During fresh install of the NVIDIA GPU Operator with time-slicing enabled.
+ ```bash
+ helm install gpu-operator nvidia/gpu-operator \
+ -n nvidia \
+ --set devicePlugin.config.name=time-slicing-config
+ ```
+
+- For dynamically enabling time-slicing with GPU Operator already installed.
+ ```bash
+ kubectl patch clusterpolicy/cluster-policy \
+ -n nvidia --type merge \
+ -p '{"spec": {"devicePlugin": {"config": {"name": "time-slicing-config"}}}}'
+ ```
+
+### Applying the Time-Slicing Configuration
+There are two methods:
+- Across the cluster
+
+ Install the GPU Operator by passing the time-slicing `ConfigMap` name and the default configuration.
+ ```bash
+ kubectl patch clusterpolicy/cluster-policy \
+ -n nvidia --type merge \
+ -p '{"spec": {"devicePlugin": {"config": {"name": "time-slicing-config", "default": "rtx-3070"}}}}'
+ ```
+
+- On certain nodes
+
+ Label the node with the required time-slicing configuration in the `ConfigMap`.
+ ```bash
+ kubectl label node <node-name> nvidia.com/device-plugin.config=rtx-3070
+ ```
+
+Once the GPU Operator and Time-Slicing GPUs is installed, check the status of the pods to ensure all the containers are running and the validation is complete.
+```bash
+kubectl get pods -n nvidia
+```
+
+```bash
+NAME READY STATUS RESTARTS AGE
+gpu-feature-discovery-qbslx 2/2 Running 0 20h
+gpu-operator-7bdd8bf555-7clgv 1/1 Running 0 20h
+gpu-operator-node-feature-discovery-master-59b4b67f4f-q84zn 1/1 Running 0 20h
+gpu-operator-node-feature-discovery-worker-n58dv 1/1 Running 0 20h
+nvidia-container-toolkit-daemonset-8gv44 1/1 Running 0 20h
+nvidia-cuda-validator-tstpk 0/1 Completed 0 20h
+nvidia-dcgm-exporter-pgk7v 1/1 Running 1 20h
+nvidia-device-plugin-daemonset-w8hh4 2/2 Running 0 20h
+nvidia-device-plugin-validator-qrpxx 0/1 Completed 0 20h
+nvidia-operator-validator-htp6b 1/1 Running 0 20h
+```
+Verify that the time-slicing configuration is applied successfully.
+```bash
+kubectl describe node <node-name>
+```
+
+```bash
+...
+Capacity:
+ nvidia.com/gpu: 8
+...
+Allocatable:
+ nvidia.com/gpu: 8
+...
+```
+
+### Testing GPU Time-Slicing
+Create a wordload test file `plugin-test.yaml`.
+```yaml
+# plugin-test.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: nvidia-plugin-test
+ labels:
+ app: nvidia-plugin-test
+spec:
+ replicas: 5
+ selector:
+ matchLabels:
+ app: nvidia-plugin-test
+ template:
+ metadata:
+ labels:
+ app: nvidia-plugin-test
+ spec:
+ tolerations:
+ - key: nvidia.com/gpu
+ operator: Exists
+ effect: NoSchedule
+ containers:
+ - name: dcgmproftester11
+ image: nvidia/samples:dcgmproftester-2.1.7-cuda11.2.2-ubuntu20.04
+ command: ["/bin/sh", "-c"]
+ args:
+ - while true; do /usr/bin/dcgmproftester11 --no-dcgm-validation -t 1004 -d 300; sleep 30; done
+ resources:
+ limits:
+ nvidia.com/gpu: 1
+ securityContext:
+ capabilities:
+ add: ["SYS_ADMIN"]
+```
+
+Create a deployment with multiple replicas.
+```bash
+kubectl apply -f plugin-test.yaml
+```
+
+Verify that all five replicas are running.
+- In pods
+ ```bash
+ kubectl get pods
+ ```
+
+ ```bash
+ NAME READY STATUS RESTARTS AGE
+ nvidia-plugin-test-677775d6c5-bpsvn 1/1 Running 0 8m8s
+ nvidia-plugin-test-677775d6c5-m95zm 1/1 Running 0 8m8s
+ nvidia-plugin-test-677775d6c5-9kgzg 1/1 Running 0 8m8s
+ nvidia-plugin-test-677775d6c5-lrl2c 1/1 Running 0 8m8s
+ nvidia-plugin-test-677775d6c5-9r2pz 1/1 Running 0 8m8s
+ ```
+- In node
+ ```bash
+ kubectl describe node <node-name>
+ ```
+
+ ```bash
+ ...
+ Allocated resources:
+ (Total limits may be over 100 percent, i.e., overcommitted.)
+ Resource Requests Limits
+ -------- -------- ------
+ ...
+ nvidia.com/gpu 5 5
+ ...
+ ```
+- In NVIDIA system management Interface
+ ```bash
+ nvidia-smi
+ ```
+
+ ```bash
+ +-----------------------------------------------------------------------------+
+ | NVIDIA-SMI 520.61.05 Driver Version: 520.61.05 CUDA Version: 11.8 |
+ |-------------------------------+----------------------+----------------------+
+ | GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
+ | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
+ | | | MIG M. |
+ |===============================+======================+======================|
+ | 0 NVIDIA GeForce ... On | 00000000:01:00.0 On | N/A |
+ | 46% 86C P2 214W / 220W | 4297MiB / 8192MiB | 100% Default |
+ | | | N/A |
+ +-------------------------------+----------------------+----------------------+
+
+ +-----------------------------------------------------------------------------+
+ | Processes: |
+ | GPU GI CI PID Type Process name GPU Memory |
+ | ID ID Usage |
+ |=============================================================================|
+ | 0 N/A N/A 1776886 C /usr/bin/dcgmproftester11 764MiB |
+ | 0 N/A N/A 1776921 C /usr/bin/dcgmproftester11 764MiB |
+ | 0 N/A N/A 1776937 C /usr/bin/dcgmproftester11 764MiB |
+ | 0 N/A N/A 1777068 C /usr/bin/dcgmproftester11 764MiB |
+ | 0 N/A N/A 1777079 C /usr/bin/dcgmproftester11 764MiB |
+ +-----------------------------------------------------------------------------+
+ ```
+
+- In Yunikorn UI applications
+
diff --git a/versioned_docs/version-1.5.0/user_guide/workloads/run_spark.md b/versioned_docs/version-1.5.0/user_guide/workloads/run_spark.md
new file mode 100644
index 0000000..e5e1373
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/workloads/run_spark.md
@@ -0,0 +1,166 @@
+---
+id: run_spark
+title: Run Spark Jobs
+description: How to run Spark jobs with YuniKorn
+keywords:
+ - spark
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+:::note
+This document assumes you have YuniKorn and its admission-controller both installed. Please refer to
+[get started](../../get_started/get_started.md) to see how that is done.
+:::
+
+## Prepare the docker image for Spark
+
+To run Spark on Kubernetes, you'll need the Spark docker images. You can 1) use the docker images provided by the Spark
+team, or 2) build one from scratch.
+If you want to build your own Spark docker image, you can find the [full instructions](https://spark.apache.org/docs/latest/building-spark.html)
+in the Spark documentation. Simplified steps:
+* Download a Spark version that has Kubernetes support, URL: https://github.com/apache/spark
+* Build spark with Kubernetes support:
+```shell script
+./build/mvn -Pkubernetes -DskipTests clean package
+```
+Recommendation is to use the official images with different spark versions in the [dockerhub](https://hub.docker.com/r/apache/spark/tags)
+
+
+## Create a namespace for Spark jobs
+
+Create a namespace:
+
+```shell script
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: spark-test
+EOF
+```
+
+## Create service account and role binding
+
+Create service account and role bindings inside the `spark-test` namespace:
+
+```shell script
+cat <<EOF | kubectl apply -n spark-test -f -
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: spark
+ namespace: spark-test
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: spark-role
+ namespace: spark-test
+rules:
+- apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["get", "watch", "list", "create", "delete"]
+- apiGroups: [""]
+ resources: ["configmaps"]
+ verbs: ["get", "create", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: spark-role-binding
+ namespace: spark-test
+subjects:
+- kind: ServiceAccount
+ name: spark
+ namespace: spark-test
+roleRef:
+ kind: Role
+ name: spark-role
+ apiGroup: rbac.authorization.k8s.io
+EOF
+```
+
+:::note
+Do NOT use `ClusterRole` and `ClusterRoleBinding` to run Spark jobs in production, please configure a more fine-grained
+security context for running Spark jobs. See more about how to configure proper RBAC rules [here](https://kubernetes.io/docs/reference/access-authn-authz/rbac/).
+:::
+
+## Submit a Spark job
+
+If this is running from local machine, you will need to start the proxy in order to talk to the api-server.
+```shell script
+kubectl proxy
+```
+
+There are official images with different spark versions in the [dockerhub](https://hub.docker.com/r/apache/spark/tags)
+Run a simple SparkPi job, this assumes that the Spark binaries are installed locally in the `/usr/local` directory.
+```shell script
+export SPARK_HOME=/usr/local/spark/
+${SPARK_HOME}/bin/spark-submit --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
+ --master k8s://http://localhost:8001 --deploy-mode cluster --name spark-pi \
+ --class org.apache.spark.examples.SparkPi \
+ --conf spark.executor.instances=1 \
+ --conf spark.kubernetes.namespace=spark-test \
+ --conf spark.kubernetes.executor.request.cores=1 \
+ --conf spark.kubernetes.container.image=docker.io/apache/spark:v3.3.0 \
+ --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
+ local:///opt/spark/examples/jars/spark-examples_2.12-3.3.0.jar
+```
+
+:::note
+There are more options for setting the driver and executor in the [spark](https://spark.apache.org/docs/latest/running-on-kubernetes.html#configuration).
+Assigning the applicationId and the queue path are possible.
+```
+--conf spark.kubernetes.executor.label.applicationId=application-spark-0001
+--conf spark.kubernetes.driver.label.applicationId=application-spark-0001
+--conf spark.kubernetes.executor.label.queue=root.default.sandbox
+--conf spark.kubernetes.driver.label.queue=root.default.sandbox
+```
+:::
+
+You'll see Spark driver and executors been created on Kubernetes:
+
+
+
+The spark-pi result is in the driver pod.
+
+
+
+## What happens behind the scenes?
+
+When the Spark job is submitted to the cluster, the job is submitted to `spark-test` namespace. The Spark driver pod will
+be firstly created under this namespace. Since this cluster has YuniKorn admission-controller enabled, when the driver pod
+get created, the admission-controller mutates the pod's spec and injects `schedulerName=yunikorn`, by doing this, the
+default K8s scheduler will skip this pod and it will be scheduled by YuniKorn instead. See how this is done by [configuring
+another scheduler in Kubernetes](https://kubernetes.io/docs/tasks/extend-kubernetes/configure-multiple-schedulers/).
+
+The default configuration has placement rule enabled, which automatically maps the `spark-test` namespace to a YuniKorn
+queue `root.spark-test`. All Spark jobs submitted to this namespace will be automatically submitted to the queue first.
+To see more about how placement rule works, please see doc [placement-rules](user_guide/placement_rules.md). By far,
+the namespace defines the security context of the pods, and the queue determines how the job and pods will be scheduled
+with considering of job ordering, queue resource fairness, etc. Note, this is the simplest setup, which doesn't enforce
+the queue capacities. The queue is considered as having unlimited capacity.
+
+YuniKorn reuses the Spark application ID set in label `spark-app-selector`, and this job is submitted
+to YuniKorn and being considered as a job. The job is scheduled and running as there is sufficient resources in the cluster.
+YuniKorn allocates the driver pod to a node, binds the pod and starts all the containers. Once the driver pod gets started,
+it requests for a bunch of executor pods to run its tasks. Those pods will be created in the same namespace as well and
+scheduled by YuniKorn as well.
diff --git a/versioned_docs/version-1.5.0/user_guide/workloads/run_tensorflow.md b/versioned_docs/version-1.5.0/user_guide/workloads/run_tensorflow.md
new file mode 100644
index 0000000..c137575
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/workloads/run_tensorflow.md
@@ -0,0 +1,228 @@
+---
+id: run_tf
+title: Run TensorFlow Jobs
+description: How to run TensorFlow jobs with YuniKorn
+keywords:
+ - tensorflow
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+This guide gives an overview of how to set up [training-operator](https://github.com/kubeflow/training-operator)
+and how to run a Tensorflow job with YuniKorn scheduler. The training-operator is a unified training operator maintained by
+Kubeflow. It not only supports TensorFlow but also PyTorch, XGboots, etc.
+
+## Install training-operator
+You can use the following command to install training operator in kubeflow namespace by default. If you have problems with installation,
+please refer to [this doc](https://github.com/kubeflow/training-operator#installation) for details.
+```
+kubectl apply -k "github.com/kubeflow/training-operator/manifests/overlays/standalone?ref=v1.3.0"
+```
+
+## Prepare the docker image
+Before you start running a TensorFlow job on Kubernetes, you'll need to build the docker image.
+1. Download files from [deployment/examples/tfjob](https://github.com/apache/yunikorn-k8shim/tree/master/deployments/examples/tfjob)
+2. To build this docker image with the following command
+
+```
+docker build -f Dockerfile -t kubeflow/tf-dist-mnist-test:1.0 .
+```
+
+## Run a TensorFlow job
+Here is a TFJob yaml for MNIST [example](https://github.com/apache/yunikorn-k8shim/blob/master/deployments/examples/tfjob/tf-job-mnist.yaml).
+
+```yaml
+apiVersion: kubeflow.org/v1
+kind: TFJob
+metadata:
+ name: dist-mnist-for-e2e-test
+ namespace: kubeflow
+spec:
+ tfReplicaSpecs:
+ PS:
+ replicas: 2
+ restartPolicy: Never
+ template:
+ metadata:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ queue: root.sandbox
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - name: tensorflow
+ image: kubeflow/tf-dist-mnist-test:1.0
+ Worker:
+ replicas: 4
+ restartPolicy: Never
+ template:
+ metadata:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ queue: root.sandbox
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - name: tensorflow
+ image: kubeflow/tf-dist-mnist-test:1.0
+```
+Create the TFJob
+```
+kubectl create -f deployments/examples/tfjob/tf-job-mnist.yaml
+```
+You can view the job info from YuniKorn UI. If you do not know how to access the YuniKorn UI,
+please read the document [here](../../get_started/get_started.md#access-the-web-ui).
+
+
+
+## Run a TensorFlow job with GPU scheduling
+To use Time-Slicing GPU your cluster must be configured to use [GPUs and Time-Slicing GPUs](https://yunikorn.apache.org/docs/next/user_guide/workloads/run_nvidia)
+This section covers a workload test scenario to validate TFJob with Time-slicing GPU.
+
+:::note
+Verify that the time-slicing configuration is applied successfully
+```bash
+kubectl describe node
+```
+
+```bash
+Capacity:
+ nvidia.com/gpu: 8
+...
+Allocatable:
+ nvidia.com/gpu: 8
+...
+```
+:::
+
+Create a workload test file `tf-gpu.yaml`
+```yaml
+# tf-gpu.yaml
+apiVersion: "kubeflow.org/v1"
+kind: "TFJob"
+metadata:
+ name: "tf-smoke-gpu"
+ namespace: kubeflow
+spec:
+ tfReplicaSpecs:
+ PS:
+ replicas: 1
+ template:
+ metadata:
+ creationTimestamp:
+ labels:
+ applicationId: "tf_job_20200521_001"
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - args:
+ - python
+ - tf_cnn_benchmarks.py
+ - --batch_size=32
+ - --model=resnet50
+ - --variable_update=parameter_server
+ - --flush_stdout=true
+ - --num_gpus=1
+ - --local_parameter_device=cpu
+ - --device=cpu
+ - --data_format=NHWC
+ image: docker.io/kubeflow/tf-benchmarks-cpu:v20171202-bdab599-dirty-284af3
+ name: tensorflow
+ ports:
+ - containerPort: 2222
+ name: tfjob-port
+ workingDir: /opt/tf-benchmarks/scripts/tf_cnn_benchmarks
+ restartPolicy: OnFailure
+ Worker:
+ replicas: 1
+ template:
+ metadata:
+ creationTimestamp: null
+ labels:
+ applicationId: "tf_job_20200521_001"
+ spec:
+ schedulerName: yunikorn
+ containers:
+ - args:
+ - python
+ - tf_cnn_benchmarks.py
+ - --batch_size=32
+ - --model=resnet50
+ - --variable_update=parameter_server
+ - --flush_stdout=true
+ - --num_gpus=1
+ - --local_parameter_device=cpu
+ - --device=gpu
+ - --data_format=NHWC
+ image: docker.io/kubeflow/tf-benchmarks-gpu:v20171202-bdab599-dirty-284af3
+ name: tensorflow
+ ports:
+ - containerPort: 2222
+ name: tfjob-port
+ resources:
+ limits:
+ nvidia.com/gpu: 2
+ workingDir: /opt/tf-benchmarks/scripts/tf_cnn_benchmarks
+ restartPolicy: OnFailure
+```
+Create the TFJob
+```bash
+kubectl apply -f tf-gpu.yaml
+kubectl get pods -n kubeflow
+```
+```bash
+NAME READY STATUS RESTARTS AGE
+tf-smoke-gpu-ps-0 1/1 Running 0 18m
+tf-smoke-gpu-worker-0 1/1 Running 0 18m
+training-operator-7d98f9dd88-dd45l 1/1 Running 0 19m
+```
+
+Verify that TFJob are running.
+- In pod logs
+ ```bash
+ kubectl logs tf-smoke-gpu-worker-0 -n kubeflow
+ ```
+ ```
+ .......
+ ..Found device 0 with properties
+ ..name: NVIDIA GeForce RTX 3080 major: 8 minor: 6 memoryClockRate(GHz): 1.71
+
+ .......
+ ..Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: NVIDIA GeForce RTX 3080, pci bus id: 0000:01:00.0, compute capability: 8.6)
+ .......
+ ```
+
+- In node
+ ```bash
+ ...
+ Allocated resources:
+ (Total limits may be over 100 percent, i.e., overcommitted.)
+ Resource Requests Limits
+ -------- -------- ------
+ ...
+ nvidia.com/gpu 2 2
+ ...
+ ```
+
+- In Yunikorn UI applications
+ 
+
+
+
diff --git a/versioned_docs/version-1.5.0/user_guide/workloads/workload_overview.md b/versioned_docs/version-1.5.0/user_guide/workloads/workload_overview.md
new file mode 100644
index 0000000..7040e79
--- /dev/null
+++ b/versioned_docs/version-1.5.0/user_guide/workloads/workload_overview.md
@@ -0,0 +1,60 @@
+---
+id: workload_overview
+title: Overview
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+The YuniKorn scheduler is able to run any Kubernetes workload. All that is required is to ensure
+that the `schedulerName` field of a Pod specification is set to `yunikorn` and an `applicationId`
+label is set to a unique value per application:
+
+```yaml
+apiVersion: v1
+kind: Pod
+metadata:
+ labels:
+ app: sleep
+ applicationId: "application-sleep-0001"
+ name: sleep-app-1
+spec:
+ schedulerName: yunikorn
+ containers:
+ - name: sleep-30s
+ image: "alpine:latest"
+ command: ["sleep", "30"]
+ resources:
+ requests:
+ cpu: "100m"
+ memory: "100M"
+```
+
+Additionally, if the YuniKorn admission controller is present, the `schedulerName` field may be
+omitted as it will be set automatically on newly created pods.
+
+## Advanced Examples
+
+Examples of more advanced use cases can be found here:
+
+* [Run NVIDIA GPU Jobs](run_nvidia)
+* [Run Spark Jobs](run_spark)
+* [Run Flink Jobs](run_flink)
+* [Run TensorFlow Jobs](run_tf)
+* [Run MPI Jobs](run_mpi)
diff --git a/versioned_sidebars/version-1.5.0-sidebars.json b/versioned_sidebars/version-1.5.0-sidebars.json
new file mode 100644
index 0000000..5bd1149
--- /dev/null
+++ b/versioned_sidebars/version-1.5.0-sidebars.json
@@ -0,0 +1,106 @@
+{
+ "docs": {
+ "Get Started": [
+ "get_started/user_guide",
+ "get_started/core_features",
+ "get_started/version"
+ ],
+ "User Guide": [
+ "user_guide/deployment_modes",
+ "user_guide/service_config",
+ "user_guide/queue_config",
+ "user_guide/placement_rules",
+ "user_guide/usergroup_resolution",
+ "user_guide/sorting_policies",
+ "user_guide/priorities",
+ "user_guide/preemption_cases",
+ "user_guide/acls",
+ "user_guide/resource_quota_management",
+ "user_guide/gang_scheduling",
+ "user_guide/labels_and_annotations_in_yunikorn",
+ "user_guide/prometheus",
+ "user_guide/use_cases",
+ {
+ "type": "category",
+ "label": "Workloads",
+ "items": [
+ "user_guide/workloads/workload_overview",
+ "user_guide/workloads/run_nvidia",
+ "user_guide/workloads/run_spark",
+ "user_guide/workloads/run_flink",
+ "user_guide/workloads/run_tf",
+ "user_guide/workloads/run_mpi"
+ ]
+ },
+ {
+ "type": "category",
+ "label": "REST APIs",
+ "items": [
+ "api/cluster",
+ "api/scheduler",
+ "api/system"
+ ]
+ },
+ {
+ "type": "category",
+ "label": "Metrics for Prometheus",
+ "items": [
+ "metrics/scheduler",
+ "metrics/runtime",
+ "metrics/queue"
+ ]
+ },
+ "user_guide/troubleshooting"
+ ],
+ "Developer Guide": [
+ "developer_guide/env_setup",
+ "developer_guide/build",
+ "developer_guide/dependencies",
+ "developer_guide/deployment",
+ "developer_guide/openshift_development",
+ "developer_guide/scheduler_object_states",
+ {
+ "type": "category",
+ "label": "Designs",
+ "items": [
+ "design/architecture",
+ "design/scheduler_plugin",
+ "design/gang_scheduling",
+ "design/user_group",
+ "design/user_based_resource_usage_tracking",
+ "design/user_group_resource_usage_enforcement",
+ "design/historical_usage_tracking",
+ "design/interface_message_simplification",
+ "design/cache_removal",
+ "design/preemption",
+ "design/simple_preemptor",
+ "design/generic_resource",
+ "design/priority_scheduling",
+ "design/resilience",
+ "design/state_aware_scheduling",
+ "design/config_v2",
+ "design/scheduler_configuration"
+ ]
+ },
+ {
+ "type": "category",
+ "label": "Archived Designs",
+ "items": [
+ "archived_design/k8shim",
+ "archived_design/namespace_resource_quota",
+ "archived_design/predicates",
+ "archived_design/scheduler_core_design",
+ "archived_design/cross_queue_preemption",
+ "archived_design/pluggable_app_management"
+ ]
+ }
+ ],
+ "Performance": [
+ "performance/evaluate_perf_function_with_kubemark",
+ "performance/evaluate_perf_function_with_kwok",
+ "performance/performance_tutorial",
+ "performance/metrics",
+ "performance/profiling"
+ ]
+ }
+}
diff --git a/versions.json b/versions.json
index 2cc43b5..464bb73 100644
--- a/versions.json
+++ b/versions.json
@@ -1,4 +1,5 @@
[
+ "1.5.0",
"1.4.0",
"1.3.0",
"1.2.0",
