| { |
| "cells": [ |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "## Sample for Submarine Experiment SDK\n", |
| "\n", |
| "The notebook shows how to use Submarine Experiment SDK to create, get, list, log, delete Submarine Experiment." |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 1, |
| "metadata": {}, |
| "outputs": [], |
| "source": [ |
| "from __future__ import print_function\n", |
| "import submarine\n", |
| "from submarine.experiment.models.environment_spec import EnvironmentSpec\n", |
| "from submarine.experiment.models.experiment_spec import ExperimentSpec\n", |
| "from submarine.experiment.models.experiment_task_spec import ExperimentTaskSpec\n", |
| "from submarine.experiment.models.experiment_meta import ExperimentMeta\n", |
| "from submarine.experiment.models.code_spec import CodeSpec" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Create Submarine Client" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "outputs": [], |
| "source": [ |
| "# Host is \"<SUBMARINE_SERVER_DNS_NAME>:<SUBMARINE_SERVER_PORT>\" by default\n", |
| "# If you don't wanna use Jupyter notebook which generated by Submarine,\n", |
| "# please set the two environment variables \"SUBMARINE_SERVER_DNS_NAME\"\n", |
| "# (e.q. localhost) and \"SUBMARINE_SERVER_PORT\" (e.q. 8080)." |
| ], |
| "metadata": { |
| "collapsed": false, |
| "pycharm": { |
| "name": "#%%\n" |
| } |
| } |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 2, |
| "metadata": { |
| "pycharm": { |
| "name": "#%%\n" |
| } |
| }, |
| "outputs": [], |
| "source": [ |
| "submarine_client = submarine.ExperimentClient()" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": { |
| "pycharm": { |
| "name": "#%% md\n" |
| } |
| }, |
| "source": [ |
| "### Define TensorFlow experiment spec¶\n", |
| "Define Submarine spec¶\n", |
| "The demo only creates a PS and worker of TF experiment to run mnist sample." |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 3, |
| "metadata": { |
| "pycharm": { |
| "name": "#%%\n" |
| } |
| }, |
| "outputs": [], |
| "source": [ |
| "environment = EnvironmentSpec(image='apache/submarine:tf-dist-mnist-test-1.0')\n", |
| "experiment_meta = ExperimentMeta(name='mnist-dist',\n", |
| " namespace='default',\n", |
| " framework='Tensorflow',\n", |
| " cmd='python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n", |
| " env_vars={'ENV1': 'ENV1'})\n", |
| "\n", |
| "worker_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M',\n", |
| " replicas=1)\n", |
| "ps_spec = ExperimentTaskSpec(resources='cpu=1,memory=1024M',\n", |
| " replicas=1)\n", |
| "code_spec = CodeSpec(sync_mode='git', url='https://github.com/apache/submarine.git')\n", |
| "\n", |
| "experiment_spec = ExperimentSpec(meta=experiment_meta,\n", |
| " environment=environment,\n", |
| " code=code_spec,\n", |
| " spec={'Ps' : ps_spec,'Worker': worker_spec})\n" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Create experiment" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 4, |
| "metadata": { |
| "pycharm": { |
| "name": "#%%\n" |
| }, |
| "scrolled": true |
| }, |
| "outputs": [ |
| { |
| "data": { |
| "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Accepted',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': None,\n 'runningTime': None,\n 'finishedTime': None,\n 'spec': {'meta': {'name': 'mnist-dist',\n 'namespace': 'default',\n 'framework': 'Tensorflow',\n 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n 'envVars': {'ENV1': 'ENV1'}},\n 'environment': {'name': None,\n 'dockerImage': None,\n 'kernelSpec': None,\n 'description': None,\n 'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n 'spec': {'Ps': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n 'Worker': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n 'code': {'syncMode': 'git',\n 'url': 'https://github.com/apache/submarine.git'}}}" |
| }, |
| "execution_count": 4, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "source": [ |
| "experiment = submarine_client.create_experiment(experiment_spec=experiment_spec)\n", |
| "experiment" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Get the created experiment" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 5, |
| "metadata": { |
| "pycharm": { |
| "name": "#%%\n" |
| } |
| }, |
| "outputs": [ |
| { |
| "data": { |
| "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Created',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': '2020-09-25T16:52:17.000+08:00',\n 'runningTime': None,\n 'finishedTime': None,\n 'spec': {'meta': {'name': 'mnist-dist',\n 'namespace': 'default',\n 'framework': 'Tensorflow',\n 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n 'envVars': {'ENV1': 'ENV1'}},\n 'environment': {'name': None,\n 'dockerImage': None,\n 'kernelSpec': None,\n 'description': None,\n 'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n 'spec': {'Ps': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n 'Worker': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n 'code': {'syncMode': 'git',\n 'url': 'https://github.com/apache/submarine.git'}}}" |
| }, |
| "execution_count": 5, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "source": [ |
| "id = experiment['experimentId']\n", |
| "submarine_client.get_experiment(id)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### List all running experiments" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 8, |
| "metadata": { |
| "pycharm": { |
| "name": "#%%\n" |
| }, |
| "scrolled": true |
| }, |
| "outputs": [ |
| { |
| "data": { |
| "text/plain": "[{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Running',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': '2020-09-25T16:52:17.000+08:00',\n 'runningTime': '2020-09-25T16:53:19.000+08:00',\n 'finishedTime': None,\n 'spec': {'meta': {'name': 'mnist-dist',\n 'namespace': 'default',\n 'framework': 'Tensorflow',\n 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n 'envVars': {'ENV1': 'ENV1'}},\n 'environment': {'name': None,\n 'dockerImage': None,\n 'kernelSpec': None,\n 'description': None,\n 'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n 'spec': {'Ps': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n 'Worker': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n 'code': {'syncMode': 'git',\n 'url': 'https://github.com/apache/submarine.git'}}}]" |
| }, |
| "execution_count": 8, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "source": [ |
| "status = 'running'\n", |
| "submarine_client.list_experiments(status=status)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Wait for the experiment to finish" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 9, |
| "metadata": { |
| "scrolled": false |
| }, |
| "outputs": [ |
| { |
| "name": "stderr", |
| "output_type": "stream", |
| "text": [ |
| "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", |
| " from ._conv import register_converters as _register_converters\n", |
| "2020-09-25 08:53:11.824375: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n", |
| "2020-09-25 08:53:11.832165: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n", |
| "2020-09-25 08:53:11.832195: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> mnist-dist-worker-0.default.svc:2222}\n", |
| "2020-09-25 08:53:11.878806: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n" |
| ] |
| } |
| ], |
| "source": [ |
| "submarine_client.wait_for_finish(id)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Get specific experiment training log" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 10, |
| "metadata": { |
| "pycharm": { |
| "name": "#%%\n" |
| }, |
| "scrolled": true |
| }, |
| "outputs": [ |
| { |
| "name": "stderr", |
| "output_type": "stream", |
| "text": [ |
| "The logs of Pod mnist-dist-ps-0:\n", |
| "\n", |
| "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", |
| " from ._conv import register_converters as _register_converters\n", |
| "2020-09-25 08:53:11.824375: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n", |
| "2020-09-25 08:53:11.832165: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n", |
| "2020-09-25 08:53:11.832195: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> mnist-dist-worker-0.default.svc:2222}\n", |
| "2020-09-25 08:53:11.878806: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n", |
| "The logs of Pod mnist-dist-worker-0:\n", |
| "\n", |
| "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", |
| " from ._conv import register_converters as _register_converters\n", |
| "2020-09-25 08:53:21.614236: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n", |
| "2020-09-25 08:53:21.622645: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> mnist-dist-ps-0.default.svc:2222}\n", |
| "2020-09-25 08:53:21.622666: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> localhost:2222}\n", |
| "2020-09-25 08:53:21.627061: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n", |
| "WARNING:tensorflow:From /var/tf_dist_mnist/dist_mnist.py:239: __init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version.\n", |
| "Instructions for updating:\n", |
| "Please switch to tf.train.MonitoredTrainingSession\n", |
| "2020-09-25 08:53:21.893834: I tensorflow/core/distributed_runtime/master_session.cc:1017] Start master session 1f6c5256b60e4b78 with config: device_filters: \"/job:ps\" device_filters: \"/job:worker/task:0\" allow_soft_placement: true\n", |
| "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n", |
| "Extracting /tmp/mnist-data/train-images-idx3-ubyte.gz\n", |
| "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n", |
| "Extracting /tmp/mnist-data/train-labels-idx1-ubyte.gz\n", |
| "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n", |
| "Extracting /tmp/mnist-data/t10k-images-idx3-ubyte.gz\n", |
| "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n", |
| "Extracting /tmp/mnist-data/t10k-labels-idx1-ubyte.gz\n", |
| "job name = worker\n", |
| "task index = 0\n", |
| "Worker 0: Initializing session...\n", |
| "Worker 0: Session initialization complete.\n", |
| "Training begins @ 1601024002.812838\n", |
| "1601024003.476335: Worker 0: training step 1 done (global step: 0)\n", |
| "1601024004.576477: Worker 0: training step 2 done (global step: 1)\n", |
| "1601024005.576241: Worker 0: training step 3 done (global step: 2)\n", |
| "1601024006.076146: Worker 0: training step 4 done (global step: 3)\n", |
| "1601024007.176262: Worker 0: training step 5 done (global step: 4)\n", |
| "1601024007.475472: Worker 0: training step 6 done (global step: 5)\n", |
| "1601024007.582544: Worker 0: training step 7 done (global step: 6)\n", |
| "1601024007.681255: Worker 0: training step 8 done (global step: 7)\n", |
| "1601024007.788009: Worker 0: training step 9 done (global step: 8)\n", |
| "1601024007.881521: Worker 0: training step 10 done (global step: 9)\n", |
| "1601024008.080983: Worker 0: training step 11 done (global step: 10)\n", |
| "1601024008.183470: Worker 0: training step 12 done (global step: 11)\n", |
| "1601024008.579565: Worker 0: training step 13 done (global step: 12)\n", |
| "1601024008.681914: Worker 0: training step 14 done (global step: 13)\n", |
| "1601024009.079380: Worker 0: training step 15 done (global step: 14)\n", |
| "1601024009.678963: Worker 0: training step 16 done (global step: 15)\n", |
| "1601024010.179325: Worker 0: training step 17 done (global step: 16)\n", |
| "1601024010.278798: Worker 0: training step 18 done (global step: 17)\n", |
| "1601024010.481322: Worker 0: training step 19 done (global step: 18)\n", |
| "1601024011.476729: Worker 0: training step 20 done (global step: 19)\n", |
| "1601024011.976693: Worker 0: training step 21 done (global step: 20)\n", |
| "1601024012.376771: Worker 0: training step 22 done (global step: 21)\n", |
| "1601024012.679685: Worker 0: training step 23 done (global step: 22)\n", |
| "1601024012.690023: Worker 0: training step 24 done (global step: 23)\n", |
| "1601024013.385506: Worker 0: training step 25 done (global step: 24)\n", |
| "1601024013.679623: Worker 0: training step 26 done (global step: 25)\n", |
| "1601024013.880934: Worker 0: training step 27 done (global step: 26)\n", |
| "1601024014.179751: Worker 0: training step 28 done (global step: 27)\n", |
| "1601024014.379176: Worker 0: training step 29 done (global step: 28)\n", |
| "1601024014.580685: Worker 0: training step 30 done (global step: 29)\n", |
| "1601024014.876878: Worker 0: training step 31 done (global step: 30)\n", |
| "1601024014.980178: Worker 0: training step 32 done (global step: 31)\n", |
| "1601024015.778837: Worker 0: training step 33 done (global step: 32)\n", |
| "1601024015.876221: Worker 0: training step 34 done (global step: 33)\n", |
| "1601024016.176451: Worker 0: training step 35 done (global step: 34)\n", |
| "1601024016.579983: Worker 0: training step 36 done (global step: 35)\n", |
| "1601024016.679807: Worker 0: training step 37 done (global step: 36)\n", |
| "1601024016.780042: Worker 0: training step 38 done (global step: 37)\n", |
| "1601024016.879108: Worker 0: training step 39 done (global step: 38)\n", |
| "1601024017.579612: Worker 0: training step 40 done (global step: 39)\n", |
| "1601024017.985924: Worker 0: training step 41 done (global step: 40)\n", |
| "1601024018.375762: Worker 0: training step 42 done (global step: 41)\n", |
| "1601024018.582078: Worker 0: training step 43 done (global step: 42)\n", |
| "1601024018.679796: Worker 0: training step 44 done (global step: 43)\n", |
| "1601024018.779481: Worker 0: training step 45 done (global step: 44)\n", |
| "1601024018.876627: Worker 0: training step 46 done (global step: 45)\n", |
| "1601024018.889815: Worker 0: training step 47 done (global step: 46)\n", |
| "1601024019.280158: Worker 0: training step 48 done (global step: 47)\n", |
| "1601024019.289733: Worker 0: training step 49 done (global step: 48)\n", |
| "1601024019.779576: Worker 0: training step 50 done (global step: 49)\n", |
| "1601024020.182737: Worker 0: training step 51 done (global step: 50)\n", |
| "1601024020.381563: Worker 0: training step 52 done (global step: 51)\n", |
| "1601024020.780347: Worker 0: training step 53 done (global step: 52)\n", |
| "1601024021.279405: Worker 0: training step 54 done (global step: 53)\n", |
| "1601024021.376435: Worker 0: training step 55 done (global step: 54)\n", |
| "1601024021.479132: Worker 0: training step 56 done (global step: 55)\n", |
| "1601024021.495602: Worker 0: training step 57 done (global step: 56)\n", |
| "1601024021.777862: Worker 0: training step 58 done (global step: 57)\n", |
| "1601024022.376070: Worker 0: training step 59 done (global step: 58)\n", |
| "1601024022.679886: Worker 0: training step 60 done (global step: 59)\n", |
| "1601024022.779463: Worker 0: training step 61 done (global step: 60)\n", |
| "1601024022.982541: Worker 0: training step 62 done (global step: 61)\n", |
| "1601024023.577791: Worker 0: training step 63 done (global step: 62)\n", |
| "1601024023.983038: Worker 0: training step 64 done (global step: 63)\n", |
| "1601024024.577057: Worker 0: training step 65 done (global step: 64)\n", |
| "1601024024.684178: Worker 0: training step 66 done (global step: 65)\n", |
| "1601024024.779040: Worker 0: training step 67 done (global step: 66)\n", |
| "1601024024.879844: Worker 0: training step 68 done (global step: 67)\n", |
| "1601024025.077337: Worker 0: training step 69 done (global step: 68)\n", |
| "1601024025.381255: Worker 0: training step 70 done (global step: 69)\n", |
| "1601024025.488024: Worker 0: training step 71 done (global step: 70)\n", |
| "1601024025.776012: Worker 0: training step 72 done (global step: 71)\n", |
| "1601024026.479723: Worker 0: training step 73 done (global step: 72)\n", |
| "1601024026.884347: Worker 0: training step 74 done (global step: 73)\n", |
| "1601024027.175786: Worker 0: training step 75 done (global step: 74)\n", |
| "1601024027.380380: Worker 0: training step 76 done (global step: 75)\n", |
| "1601024027.389443: Worker 0: training step 77 done (global step: 76)\n", |
| "1601024027.481207: Worker 0: training step 78 done (global step: 77)\n", |
| "1601024027.982406: Worker 0: training step 79 done (global step: 78)\n", |
| "1601024028.081975: Worker 0: training step 80 done (global step: 79)\n", |
| "1601024028.577323: Worker 0: training step 81 done (global step: 80)\n", |
| "1601024028.876491: Worker 0: training step 82 done (global step: 81)\n", |
| "1601024028.978793: Worker 0: training step 83 done (global step: 82)\n", |
| "1601024029.076089: Worker 0: training step 84 done (global step: 83)\n", |
| "1601024029.086477: Worker 0: training step 85 done (global step: 84)\n", |
| "1601024029.385806: Worker 0: training step 86 done (global step: 85)\n", |
| "1601024029.479636: Worker 0: training step 87 done (global step: 86)\n", |
| "1601024029.582871: Worker 0: training step 88 done (global step: 87)\n", |
| "1601024029.681764: Worker 0: training step 89 done (global step: 88)\n", |
| "1601024029.776664: Worker 0: training step 90 done (global step: 89)\n", |
| "1601024029.790441: Worker 0: training step 91 done (global step: 90)\n", |
| "1601024029.883305: Worker 0: training step 92 done (global step: 91)\n", |
| "1601024030.083825: Worker 0: training step 93 done (global step: 92)\n", |
| "1601024030.278688: Worker 0: training step 94 done (global step: 93)\n", |
| "1601024030.378878: Worker 0: training step 95 done (global step: 94)\n", |
| "1601024030.479512: Worker 0: training step 96 done (global step: 95)\n", |
| "1601024030.579052: Worker 0: training step 97 done (global step: 96)\n", |
| "1601024030.981217: Worker 0: training step 98 done (global step: 97)\n", |
| "1601024031.175745: Worker 0: training step 99 done (global step: 98)\n", |
| "1601024031.479927: Worker 0: training step 100 done (global step: 99)\n", |
| "1601024031.882804: Worker 0: training step 101 done (global step: 100)\n", |
| "Training ends @ 1601024031.883017\n", |
| "Training elapsed time: 29.070179 s\n", |
| "After 100 training step(s), validation cross entropy = 1220.89\n" |
| ] |
| } |
| ], |
| "source": [ |
| "submarine_client.get_log(id)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Delete the experiment" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 11, |
| "metadata": { |
| "pycharm": { |
| "name": "#%%\n" |
| } |
| }, |
| "outputs": [ |
| { |
| "data": { |
| "text/plain": "{'experimentId': 'experiment_1601021036429_0013',\n 'name': 'mnist-dist',\n 'uid': 'fdee35f9-7877-4f59-8b19-c83fe3635408',\n 'status': 'Deleted',\n 'acceptedTime': '2020-09-25T16:52:17.000+08:00',\n 'createdTime': '2020-09-25T16:52:17.000+08:00',\n 'runningTime': '2020-09-25T16:53:19.000+08:00',\n 'finishedTime': '2020-09-25T16:53:54.000+08:00',\n 'spec': {'meta': {'name': 'mnist-dist',\n 'namespace': 'default',\n 'framework': 'Tensorflow',\n 'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n 'envVars': {'ENV1': 'ENV1'}},\n 'environment': {'name': None,\n 'dockerImage': None,\n 'kernelSpec': None,\n 'description': None,\n 'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n 'spec': {'Ps': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n 'Worker': {'replicas': 1,\n 'resources': 'cpu=1,memory=1024M',\n 'name': None,\n 'image': None,\n 'cmd': None,\n 'envVars': None,\n 'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n 'code': {'syncMode': 'git',\n 'url': 'https://github.com/apache/submarine.git'}}}" |
| }, |
| "execution_count": 11, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "source": [ |
| "submarine_client.delete_experiment(id)" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": {}, |
| "outputs": [], |
| "source": [] |
| } |
| ], |
| "metadata": { |
| "kernelspec": { |
| "display_name": "Python 3", |
| "language": "python", |
| "name": "python3" |
| }, |
| "language_info": { |
| "codemirror_mode": { |
| "name": "ipython", |
| "version": 3 |
| }, |
| "file_extension": ".py", |
| "mimetype": "text/x-python", |
| "name": "python", |
| "nbconvert_exporter": "python", |
| "pygments_lexer": "ipython3", |
| "version": "3.7.7" |
| } |
| }, |
| "nbformat": 4, |
| "nbformat_minor": 1 |
| } |
