| { |
| "cells": [ |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0. " |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "# SINGA Model Classes\n", |
| "\n", |
| "<img src=\"http://singa.apache.org/en/_static/images/singav1-sw.png\" width=\"500px\"/>" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "## Layer\n", |
| "\n", |
| "Typically, the life cycle of a layer instance includes:\n", |
| " 1. construct layer without input_sample_shapes, goto 2; or,\n", |
| " \n", |
| " construct layer with input_sample_shapes, goto 3;\n", |
| " 2. call setup to create the parameters and setup other meta fields;\n", |
| " 4. initialize the parameters of the layer\n", |
| " 3. call forward or access layer members\n", |
| " 4. call backward and get parameters for update" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 1, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "from singa import tensor, device, layer\n", |
| "\n", |
| "#help(layer.Layer)\n", |
| "layer.engine='singacpp'" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "## Common layers" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 2, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "from singa.layer import Dense, Conv2D, MaxPooling2D, Activation, BatchNormalization, Softmax" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Dense Layer" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 3, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "(2, 3) (3,)\n" |
| ] |
| } |
| ], |
| "source": [ |
| "dense = Dense('dense', 3, input_sample_shape=(2,))\n", |
| "#dense.param_names()\n", |
| "w, b = dense.param_values()\n", |
| "print(w.shape, b.shape)" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 4, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "w.gaussian(0, 0.1)\n", |
| "b.set_value(0)" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 5, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "data": { |
| "text/plain": [ |
| "array([[ 0.02440065, -0.03396009, 0.01396658],\n", |
| " [ 0.00771775, 0.07841966, -0.05931654]], dtype=float32)" |
| ] |
| }, |
| "execution_count": 5, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "source": [ |
| "x = tensor.Tensor((2,2))\n", |
| "x.uniform(-1, 1)\n", |
| "y = dense.forward(True, x)\n", |
| "tensor.to_numpy(y)\n" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 6, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "(2, 2) (2, 3) (3,)\n" |
| ] |
| } |
| ], |
| "source": [ |
| "gx, [gw, gb] = dense.backward(True, y)\n", |
| "print(gx.shape, gw.shape, gb.shape)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Convolution Layer" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 7, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "(4, 6, 6)\n" |
| ] |
| } |
| ], |
| "source": [ |
| "conv = Conv2D('conv', 4, 3, 1, input_sample_shape=(3, 6, 6))\n", |
| "print(conv.get_output_sample_shape())" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "### Pooling Layer" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 8, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "(4, 3, 3)\n" |
| ] |
| } |
| ], |
| "source": [ |
| "pool = MaxPooling2D('pool', 3, 2, input_sample_shape=(4, 6, 6))\n", |
| "print(pool.get_output_sample_shape())" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "## Branch layers" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 9, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [ |
| "from singa.layer import Split, Merge, Slice, Concat" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 10, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "[(4, 6, 6), (4, 6, 6)]\n" |
| ] |
| } |
| ], |
| "source": [ |
| "split = Split('split', 2, input_sample_shape=(4, 6, 6))\n", |
| "print(split.get_output_sample_shape())" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 11, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "(4, 6, 6)\n" |
| ] |
| } |
| ], |
| "source": [ |
| "merge = Merge('merge', input_sample_shape=(4, 6, 6))\n", |
| "print(merge.get_output_sample_shape())" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 12, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "[(2, 6, 6), (2, 6, 6)]\n" |
| ] |
| } |
| ], |
| "source": [ |
| "sli = Slice('slice', 1, [2], input_sample_shape=(4, 6, 6))\n", |
| "print(sli.get_output_sample_shape())" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 13, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "(4, 6, 6)\n" |
| ] |
| } |
| ], |
| "source": [ |
| "concat = Concat('concat', 1, input_sample_shapes=[(3, 6, 6), (1, 6, 6)])\n", |
| "print(concat.get_output_sample_shape())" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "## Metric and Loss" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 14, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "[[ 0.13973515 0.21827343 0.21068712 0.21905626 0.21224809]\n", |
| " [ 0.2354937 0.1047527 0.18490241 0.23617713 0.23867407]\n", |
| " [ 0.2659435 0.11397494 0.1659178 0.22683497 0.22732878]]\n", |
| "0.0\n" |
| ] |
| } |
| ], |
| "source": [ |
| "from singa import metric\n", |
| "import numpy as np\n", |
| "\n", |
| "x = tensor.Tensor((3, 5))\n", |
| "x.uniform(0, 1) # randomly genearte the prediction activation\n", |
| "x = tensor.softmax(x) # normalize the prediction into probabilities\n", |
| "print(tensor.to_numpy(x))\n", |
| "y = tensor.from_numpy(np.array([0, 1, 3], dtype=np.int)) # set the truth\n", |
| "\n", |
| "f = metric.Accuracy()\n", |
| "acc = f.evaluate(x, y) # averaged accuracy over all 3 samples in x\n", |
| "print(acc)" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 15, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "1.8030937910079956\n", |
| "[[-0.78104687 0.18748793 0.16346708 0.24803984 0.18205206]\n", |
| " [ 0.21501946 -0.83683592 0.19003348 0.20714596 0.22463693]\n", |
| " [ 0.20000091 0.23285127 0.26842937 -0.87474263 0.17346108]]\n" |
| ] |
| } |
| ], |
| "source": [ |
| "from singa import loss\n", |
| "\n", |
| "x = tensor.Tensor((3, 5))\n", |
| "x.uniform(0, 1) # randomly genearte the prediction activation\n", |
| "y = tensor.from_numpy(np.array([0, 1, 3], dtype=np.int)) # set the truth\n", |
| "\n", |
| "f = loss.SoftmaxCrossEntropy()\n", |
| "l = f.forward(True, x, y) # l is tensor with 3 loss values\n", |
| "g = f.backward() # g is a tensor containing all gradients of x w.r.t l\n", |
| "print(l.l1())\n", |
| "print(tensor.to_numpy(g))" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "## Optimizer" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 16, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "data": { |
| "text/plain": [ |
| "<singa.tensor.Tensor at 0x7f539260f710>" |
| ] |
| }, |
| "execution_count": 16, |
| "metadata": {}, |
| "output_type": "execute_result" |
| } |
| ], |
| "source": [ |
| "from singa import optimizer\n", |
| "\n", |
| "sgd = optimizer.SGD(lr=0.01, momentum=0.9, weight_decay=1e-4)\n", |
| "p = tensor.Tensor((3,5))\n", |
| "p.uniform(-1, 1)\n", |
| "g = tensor.Tensor((3,5))\n", |
| "g.gaussian(0, 0.01)\n", |
| "\n", |
| "sgd.apply(1, g, p, 'param') # use the global lr=0.1 for epoch 1\n", |
| "sgd.apply_with_lr(2, 0.03, g, p, 'param') # use lr=0.03 for epoch 2" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": {}, |
| "source": [ |
| "## FeedForwardNet" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 17, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "('conv1', (32, 32, 32))\n", |
| "('relu1', (32, 32, 32))\n", |
| "('pool1', (32, 16, 16))\n", |
| "('flat', (8192,))\n", |
| "('dense', (10,))\n", |
| "['conv1/weight', 'conv1/bias', 'dense/weight', 'dense/bias']\n" |
| ] |
| } |
| ], |
| "source": [ |
| "from singa import net as ffnet\n", |
| "layer.engine = 'singacpp'\n", |
| "net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())\n", |
| "net.add(layer.Conv2D('conv1', 32, 5, 1, input_sample_shape=(3,32,32,)))\n", |
| "net.add(layer.Activation('relu1'))\n", |
| "net.add(layer.MaxPooling2D('pool1', 3, 2))\n", |
| "net.add(layer.Flatten('flat'))\n", |
| "net.add(layer.Dense('dense', 10))\n", |
| "\n", |
| "# init parameters\n", |
| "for p in net.param_values():\n", |
| " if len(p.shape) == 0:\n", |
| " p.set_value(0)\n", |
| " else:\n", |
| " p.gaussian(0, 0.01)\n", |
| "print(net.param_names())" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": 18, |
| "metadata": {}, |
| "outputs": [ |
| { |
| "name": "stdout", |
| "output_type": "stream", |
| "text": [ |
| "('conv1', (32, 32, 32))\n", |
| "('relu1', (32, 32, 32))\n", |
| "('pool1', (32, 16, 16))\n", |
| "('flat', (8192,))\n", |
| "('dense', (10,))\n" |
| ] |
| } |
| ], |
| "source": [ |
| "layer.engine = 'cudnn'\n", |
| "net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())\n", |
| "net.add(layer.Conv2D('conv1', 32, 5, 1, input_sample_shape=(3,32,32,)))\n", |
| "net.add(layer.Activation('relu1'))\n", |
| "net.add(layer.MaxPooling2D('pool1', 3, 2))\n", |
| "net.add(layer.Flatten('flat'))\n", |
| "net.add(layer.Dense('dense', 10))\n", |
| "\n", |
| "# init parameters\n", |
| "for p in net.param_values():\n", |
| " if len(p.shape) == 0:\n", |
| " p.set_value(0)\n", |
| " else:\n", |
| " p.gaussian(0, 0.01)\n", |
| " \n", |
| "# move net onto gpu\n", |
| "dev = device.create_cuda_gpu()\n", |
| "net.to_device(dev)" |
| ] |
| }, |
| { |
| "cell_type": "markdown", |
| "metadata": { |
| "collapsed": true |
| }, |
| "source": [ |
| "## Next: [Simple models](./regression.ipynb)" |
| ] |
| }, |
| { |
| "cell_type": "code", |
| "execution_count": null, |
| "metadata": { |
| "collapsed": true |
| }, |
| "outputs": [], |
| "source": [] |
| } |
| ], |
| "metadata": { |
| "anaconda-cloud": {}, |
| "kernelspec": { |
| "display_name": "py3", |
| "language": "python", |
| "name": "py3" |
| }, |
| "language_info": { |
| "codemirror_mode": { |
| "name": "ipython", |
| "version": 3 |
| }, |
| "file_extension": ".py", |
| "mimetype": "text/x-python", |
| "name": "python", |
| "nbconvert_exporter": "python", |
| "pygments_lexer": "ipython3", |
| "version": "3.5.3" |
| } |
| }, |
| "nbformat": 4, |
| "nbformat_minor": 2 |
| } |