{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample for Submarine Experiment SDK\n",
    "\n",
    "The notebook shows how to use Submarine Experiment SDK to create, get, list, log, delete Submarine Experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import submarine\n",
    "from submarine.experiment.models.environment_spec import EnvironmentSpec\n",
    "from submarine.experiment.models.experiment_spec import ExperimentSpec\n",
    "from submarine.experiment.models.experiment_task_spec import ExperimentTaskSpec\n",
    "from submarine.experiment.models.experiment_meta import ExperimentMeta\n",
    "from submarine.experiment.models.code_spec import CodeSpec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Submarine Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Host is \"<SUBMARINE_SERVER_DNS_NAME>:<SUBMARINE_SERVER_PORT>\" by default\n",
    "# If you don't wanna use Jupyter notebook which generated by Submarine,\n",
    "# please set the two environment variables \"SUBMARINE_SERVER_DNS_NAME\"\n",
    "# (e.q. localhost) and \"SUBMARINE_SERVER_PORT\" (e.q. 8080)."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "submarine_client = submarine.ExperimentClient()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Define TensorFlow experiment spec¶\n",
    "Define Submarine spec¶\n",
    "The demo only creates a PS and worker of TF experiment to run mnist sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "environment = EnvironmentSpec(image='apache/submarine:tf-dist-mnist-test-1.0')\n",
    "experiment_meta = ExperimentMeta(name='mnist-dist',\n",
    "                                 namespace='default',\n",
    "                                 framework='Tensorflow',\n",
    "                                 cmd='python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n",
    "                                 env_vars={'ENV1': 'ENV1'})\n",
    "\n",
    "worker_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M',\n",
    "                                 replicas=1)\n",
    "ps_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M',\n",
    "                                 replicas=1)\n",
    "code_spec = CodeSpec(sync_mode='git', url='https://github.com/apache/submarine.git')\n",
    "\n",
    "experiment_spec = ExperimentSpec(meta=experiment_meta,\n",
    "                                 environment=environment,\n",
    "                                 code=code_spec,\n",
    "                                 spec={'Ps' : ps_spec,'Worker': worker_spec})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Accepted',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': None,\n 'runningTime': None,\n 'finishedTime': None,\n 'spec': {'meta': {'name': 'mnist-dist',\n   'namespace': 'default',\n   'framework': 'Tensorflow',\n   'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n   'envVars': {'ENV1': 'ENV1'}},\n  'environment': {'name': None,\n   'dockerImage': None,\n   'kernelSpec': None,\n   'description': None,\n   'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n  'spec': {'Ps': {'replicas': 1,\n    'resources': 'cpu=1,memory=1024M',\n    'name': None,\n    'image': None,\n    'cmd': None,\n    'envVars': None,\n    'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n   'Worker': {'replicas': 1,\n    'resources': 'cpu=1,memory=1024M',\n    'name': None,\n    'image': None,\n    'cmd': None,\n    'envVars': None,\n    'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n  'code': {'syncMode': 'git',\n   'url': 'https://github.com/apache/submarine.git'}}}"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiment = submarine_client.create_experiment(experiment_spec=experiment_spec)\n",
    "experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the created experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Created',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': '2020-09-25T16:52:17.000+08:00',\n 'runningTime': None,\n 'finishedTime': None,\n 'spec': {'meta': {'name': 'mnist-dist',\n   'namespace': 'default',\n   'framework': 'Tensorflow',\n   'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n   'envVars': {'ENV1': 'ENV1'}},\n  'environment': {'name': None,\n   'dockerImage': None,\n   'kernelSpec': None,\n   'description': None,\n   'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n  'spec': {'Ps': {'replicas': 1,\n    'resources': 'cpu=1,memory=1024M',\n    'name': None,\n    'image': None,\n    'cmd': None,\n    'envVars': None,\n    'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n   'Worker': {'replicas': 1,\n    'resources': 'cpu=1,memory=1024M',\n    'name': None,\n    'image': None,\n    'cmd': None,\n    'envVars': None,\n    'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n  'code': {'syncMode': 'git',\n   'url': 'https://github.com/apache/submarine.git'}}}"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id = experiment['experimentId']\n",
    "submarine_client.get_experiment(id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List all running experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": "[{'experimentId': 'experiment_1601021036429_0013',\n  'name': 'mnist-dist',\n  'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n  'status': 'Running',\n  'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n  'createdTime': '2020-09-25T16:52:17.000+08:00',\n  'runningTime': '2020-09-25T16:53:19.000+08:00',\n  'finishedTime': None,\n  'spec': {'meta': {'name': 'mnist-dist',\n    'namespace': 'default',\n    'framework': 'Tensorflow',\n    'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n    'envVars': {'ENV1': 'ENV1'}},\n   'environment': {'name': None,\n    'dockerImage': None,\n    'kernelSpec': None,\n    'description': None,\n    'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n   'spec': {'Ps': {'replicas': 1,\n     'resources': 'cpu=1,memory=1024M',\n     'name': None,\n     'image': None,\n     'cmd': None,\n     'envVars': None,\n     'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n    'Worker': {'replicas': 1,\n     'resources': 'cpu=1,memory=1024M',\n     'name': None,\n     'image': None,\n     'cmd': None,\n     'envVars': None,\n     'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n   'code': {'syncMode': 'git',\n    'url': 'https://github.com/apache/submarine.git'}}}]"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "status = 'running'\n",
    "submarine_client.list_experiments(status=status)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for the experiment to finish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "2020-09-25 08:53:11.824375: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "2020-09-25 08:53:11.832165: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n",
      "2020-09-25 08:53:11.832195: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> mnist-dist-worker-0.default.svc:2222}\n",
      "2020-09-25 08:53:11.878806: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n"
     ]
    }
   ],
   "source": [
    "submarine_client.wait_for_finish(id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get specific experiment training log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The logs of Pod mnist-dist-ps-0:\n",
      "\n",
      "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "2020-09-25 08:53:11.824375: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "2020-09-25 08:53:11.832165: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n",
      "2020-09-25 08:53:11.832195: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> mnist-dist-worker-0.default.svc:2222}\n",
      "2020-09-25 08:53:11.878806: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n",
      "The logs of Pod mnist-dist-worker-0:\n",
      "\n",
      "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "2020-09-25 08:53:21.614236: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "2020-09-25 08:53:21.622645: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> mnist-dist-ps-0.default.svc:2222}\n",
      "2020-09-25 08:53:21.622666: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> localhost:2222}\n",
      "2020-09-25 08:53:21.627061: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n",
      "WARNING:tensorflow:From /var/tf_dist_mnist/dist_mnist.py:239: __init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.train.MonitoredTrainingSession\n",
      "2020-09-25 08:53:21.893834: I tensorflow/core/distributed_runtime/master_session.cc:1017] Start master session 1f6c5256b60e4b78 with config: device_filters: \"/job:ps\" device_filters: \"/job:worker/task:0\" allow_soft_placement: true\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting /tmp/mnist-data/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting /tmp/mnist-data/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting /tmp/mnist-data/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting /tmp/mnist-data/t10k-labels-idx1-ubyte.gz\n",
      "job name = worker\n",
      "task index = 0\n",
      "Worker 0: Initializing session...\n",
      "Worker 0: Session initialization complete.\n",
      "Training begins @ 1601024002.812838\n",
      "1601024003.476335: Worker 0: training step 1 done (global step: 0)\n",
      "1601024004.576477: Worker 0: training step 2 done (global step: 1)\n",
      "1601024005.576241: Worker 0: training step 3 done (global step: 2)\n",
      "1601024006.076146: Worker 0: training step 4 done (global step: 3)\n",
      "1601024007.176262: Worker 0: training step 5 done (global step: 4)\n",
      "1601024007.475472: Worker 0: training step 6 done (global step: 5)\n",
      "1601024007.582544: Worker 0: training step 7 done (global step: 6)\n",
      "1601024007.681255: Worker 0: training step 8 done (global step: 7)\n",
      "1601024007.788009: Worker 0: training step 9 done (global step: 8)\n",
      "1601024007.881521: Worker 0: training step 10 done (global step: 9)\n",
      "1601024008.080983: Worker 0: training step 11 done (global step: 10)\n",
      "1601024008.183470: Worker 0: training step 12 done (global step: 11)\n",
      "1601024008.579565: Worker 0: training step 13 done (global step: 12)\n",
      "1601024008.681914: Worker 0: training step 14 done (global step: 13)\n",
      "1601024009.079380: Worker 0: training step 15 done (global step: 14)\n",
      "1601024009.678963: Worker 0: training step 16 done (global step: 15)\n",
      "1601024010.179325: Worker 0: training step 17 done (global step: 16)\n",
      "1601024010.278798: Worker 0: training step 18 done (global step: 17)\n",
      "1601024010.481322: Worker 0: training step 19 done (global step: 18)\n",
      "1601024011.476729: Worker 0: training step 20 done (global step: 19)\n",
      "1601024011.976693: Worker 0: training step 21 done (global step: 20)\n",
      "1601024012.376771: Worker 0: training step 22 done (global step: 21)\n",
      "1601024012.679685: Worker 0: training step 23 done (global step: 22)\n",
      "1601024012.690023: Worker 0: training step 24 done (global step: 23)\n",
      "1601024013.385506: Worker 0: training step 25 done (global step: 24)\n",
      "1601024013.679623: Worker 0: training step 26 done (global step: 25)\n",
      "1601024013.880934: Worker 0: training step 27 done (global step: 26)\n",
      "1601024014.179751: Worker 0: training step 28 done (global step: 27)\n",
      "1601024014.379176: Worker 0: training step 29 done (global step: 28)\n",
      "1601024014.580685: Worker 0: training step 30 done (global step: 29)\n",
      "1601024014.876878: Worker 0: training step 31 done (global step: 30)\n",
      "1601024014.980178: Worker 0: training step 32 done (global step: 31)\n",
      "1601024015.778837: Worker 0: training step 33 done (global step: 32)\n",
      "1601024015.876221: Worker 0: training step 34 done (global step: 33)\n",
      "1601024016.176451: Worker 0: training step 35 done (global step: 34)\n",
      "1601024016.579983: Worker 0: training step 36 done (global step: 35)\n",
      "1601024016.679807: Worker 0: training step 37 done (global step: 36)\n",
      "1601024016.780042: Worker 0: training step 38 done (global step: 37)\n",
      "1601024016.879108: Worker 0: training step 39 done (global step: 38)\n",
      "1601024017.579612: Worker 0: training step 40 done (global step: 39)\n",
      "1601024017.985924: Worker 0: training step 41 done (global step: 40)\n",
      "1601024018.375762: Worker 0: training step 42 done (global step: 41)\n",
      "1601024018.582078: Worker 0: training step 43 done (global step: 42)\n",
      "1601024018.679796: Worker 0: training step 44 done (global step: 43)\n",
      "1601024018.779481: Worker 0: training step 45 done (global step: 44)\n",
      "1601024018.876627: Worker 0: training step 46 done (global step: 45)\n",
      "1601024018.889815: Worker 0: training step 47 done (global step: 46)\n",
      "1601024019.280158: Worker 0: training step 48 done (global step: 47)\n",
      "1601024019.289733: Worker 0: training step 49 done (global step: 48)\n",
      "1601024019.779576: Worker 0: training step 50 done (global step: 49)\n",
      "1601024020.182737: Worker 0: training step 51 done (global step: 50)\n",
      "1601024020.381563: Worker 0: training step 52 done (global step: 51)\n",
      "1601024020.780347: Worker 0: training step 53 done (global step: 52)\n",
      "1601024021.279405: Worker 0: training step 54 done (global step: 53)\n",
      "1601024021.376435: Worker 0: training step 55 done (global step: 54)\n",
      "1601024021.479132: Worker 0: training step 56 done (global step: 55)\n",
      "1601024021.495602: Worker 0: training step 57 done (global step: 56)\n",
      "1601024021.777862: Worker 0: training step 58 done (global step: 57)\n",
      "1601024022.376070: Worker 0: training step 59 done (global step: 58)\n",
      "1601024022.679886: Worker 0: training step 60 done (global step: 59)\n",
      "1601024022.779463: Worker 0: training step 61 done (global step: 60)\n",
      "1601024022.982541: Worker 0: training step 62 done (global step: 61)\n",
      "1601024023.577791: Worker 0: training step 63 done (global step: 62)\n",
      "1601024023.983038: Worker 0: training step 64 done (global step: 63)\n",
      "1601024024.577057: Worker 0: training step 65 done (global step: 64)\n",
      "1601024024.684178: Worker 0: training step 66 done (global step: 65)\n",
      "1601024024.779040: Worker 0: training step 67 done (global step: 66)\n",
      "1601024024.879844: Worker 0: training step 68 done (global step: 67)\n",
      "1601024025.077337: Worker 0: training step 69 done (global step: 68)\n",
      "1601024025.381255: Worker 0: training step 70 done (global step: 69)\n",
      "1601024025.488024: Worker 0: training step 71 done (global step: 70)\n",
      "1601024025.776012: Worker 0: training step 72 done (global step: 71)\n",
      "1601024026.479723: Worker 0: training step 73 done (global step: 72)\n",
      "1601024026.884347: Worker 0: training step 74 done (global step: 73)\n",
      "1601024027.175786: Worker 0: training step 75 done (global step: 74)\n",
      "1601024027.380380: Worker 0: training step 76 done (global step: 75)\n",
      "1601024027.389443: Worker 0: training step 77 done (global step: 76)\n",
      "1601024027.481207: Worker 0: training step 78 done (global step: 77)\n",
      "1601024027.982406: Worker 0: training step 79 done (global step: 78)\n",
      "1601024028.081975: Worker 0: training step 80 done (global step: 79)\n",
      "1601024028.577323: Worker 0: training step 81 done (global step: 80)\n",
      "1601024028.876491: Worker 0: training step 82 done (global step: 81)\n",
      "1601024028.978793: Worker 0: training step 83 done (global step: 82)\n",
      "1601024029.076089: Worker 0: training step 84 done (global step: 83)\n",
      "1601024029.086477: Worker 0: training step 85 done (global step: 84)\n",
      "1601024029.385806: Worker 0: training step 86 done (global step: 85)\n",
      "1601024029.479636: Worker 0: training step 87 done (global step: 86)\n",
      "1601024029.582871: Worker 0: training step 88 done (global step: 87)\n",
      "1601024029.681764: Worker 0: training step 89 done (global step: 88)\n",
      "1601024029.776664: Worker 0: training step 90 done (global step: 89)\n",
      "1601024029.790441: Worker 0: training step 91 done (global step: 90)\n",
      "1601024029.883305: Worker 0: training step 92 done (global step: 91)\n",
      "1601024030.083825: Worker 0: training step 93 done (global step: 92)\n",
      "1601024030.278688: Worker 0: training step 94 done (global step: 93)\n",
      "1601024030.378878: Worker 0: training step 95 done (global step: 94)\n",
      "1601024030.479512: Worker 0: training step 96 done (global step: 95)\n",
      "1601024030.579052: Worker 0: training step 97 done (global step: 96)\n",
      "1601024030.981217: Worker 0: training step 98 done (global step: 97)\n",
      "1601024031.175745: Worker 0: training step 99 done (global step: 98)\n",
      "1601024031.479927: Worker 0: training step 100 done (global step: 99)\n",
      "1601024031.882804: Worker 0: training step 101 done (global step: 100)\n",
      "Training ends @ 1601024031.883017\n",
      "Training elapsed time: 29.070179 s\n",
      "After 100 training step(s), validation cross entropy = 1220.89\n"
     ]
    }
   ],
   "source": [
    "submarine_client.get_log(id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Delete the experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Deleted',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': '2020-09-25T16:52:17.000+08:00',\n 'runningTime': '2020-09-25T16:53:19.000+08:00',\n 'finishedTime': '2020-09-25T16:53:54.000+08:00',\n 'spec': {'meta': {'name': 'mnist-dist',\n   'namespace': 'default',\n   'framework': 'Tensorflow',\n   'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n   'envVars': {'ENV1': 'ENV1'}},\n  'environment': {'name': None,\n   'dockerImage': None,\n   'kernelSpec': None,\n   'description': None,\n   'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n  'spec': {'Ps': {'replicas': 1,\n    'resources': 'cpu=1,memory=1024M',\n    'name': None,\n    'image': None,\n    'cmd': None,\n    'envVars': None,\n    'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n   'Worker': {'replicas': 1,\n    'resources': 'cpu=1,memory=1024M',\n    'name': None,\n    'image': None,\n    'cmd': None,\n    'envVars': None,\n    'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n  'code': {'syncMode': 'git',\n   'url': 'https://github.com/apache/submarine.git'}}}"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submarine_client.delete_experiment(id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}