{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SINGA Model Classes\n",
    "\n",
    "<img src=\"http://singa.apache.org/en/_static/images/singav1-sw.png\" width=\"500px\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Layer\n",
    "\n",
    "Typically, the life cycle of a layer instance includes:\n",
    " 1. construct layer without input_sample_shapes, goto 2; or,\n",
    " \n",
    "    construct layer with input_sample_shapes, goto 3;\n",
    " 2. call setup to create the parameters and setup other meta fields;\n",
    " 4. initialize the parameters of the layer\n",
    " 3. call forward or access layer members\n",
    " 4. call backward and get parameters for update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from singa import tensor, device, layer\n",
    "\n",
    "#help(layer.Layer)\n",
    "layer.engine='singacpp'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Common layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from singa.layer import Dense, Conv2D, MaxPooling2D, Activation, BatchNormalization, Softmax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dense Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 3) (3,)\n"
     ]
    }
   ],
   "source": [
    "dense = Dense('dense', 3, input_sample_shape=(2,))\n",
    "#dense.param_names()\n",
    "w, b = dense.param_values()\n",
    "print(w.shape, b.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w.gaussian(0, 0.1)\n",
    "b.set_value(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.02440065, -0.03396009,  0.01396658],\n",
       "       [ 0.00771775,  0.07841966, -0.05931654]], dtype=float32)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x  = tensor.Tensor((2,2))\n",
    "x.uniform(-1, 1)\n",
    "y = dense.forward(True, x)\n",
    "tensor.to_numpy(y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 2) (2, 3) (3,)\n"
     ]
    }
   ],
   "source": [
    "gx, [gw, gb] = dense.backward(True, y)\n",
    "print(gx.shape, gw.shape, gb.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convolution Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 6, 6)\n"
     ]
    }
   ],
   "source": [
    "conv = Conv2D('conv', 4, 3, 1, input_sample_shape=(3, 6, 6))\n",
    "print(conv.get_output_sample_shape())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pooling Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 3, 3)\n"
     ]
    }
   ],
   "source": [
    "pool = MaxPooling2D('pool', 3, 2, input_sample_shape=(4, 6, 6))\n",
    "print(pool.get_output_sample_shape())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Branch layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from singa.layer import Split, Merge, Slice, Concat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(4, 6, 6), (4, 6, 6)]\n"
     ]
    }
   ],
   "source": [
    "split = Split('split', 2, input_sample_shape=(4, 6, 6))\n",
    "print(split.get_output_sample_shape())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 6, 6)\n"
     ]
    }
   ],
   "source": [
    "merge = Merge('merge', input_sample_shape=(4, 6, 6))\n",
    "print(merge.get_output_sample_shape())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(2, 6, 6), (2, 6, 6)]\n"
     ]
    }
   ],
   "source": [
    "sli = Slice('slice', 1, [2], input_sample_shape=(4, 6, 6))\n",
    "print(sli.get_output_sample_shape())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 6, 6)\n"
     ]
    }
   ],
   "source": [
    "concat = Concat('concat', 1, input_sample_shapes=[(3, 6, 6), (1, 6, 6)])\n",
    "print(concat.get_output_sample_shape())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metric and Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.13973515  0.21827343  0.21068712  0.21905626  0.21224809]\n",
      " [ 0.2354937   0.1047527   0.18490241  0.23617713  0.23867407]\n",
      " [ 0.2659435   0.11397494  0.1659178   0.22683497  0.22732878]]\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "from singa import metric\n",
    "import numpy as np\n",
    "\n",
    "x = tensor.Tensor((3, 5))\n",
    "x.uniform(0, 1)  # randomly genearte the prediction activation\n",
    "x = tensor.softmax(x)  # normalize the prediction into probabilities\n",
    "print(tensor.to_numpy(x))\n",
    "y = tensor.from_numpy(np.array([0, 1, 3], dtype=np.int))  # set the truth\n",
    "\n",
    "f = metric.Accuracy()\n",
    "acc = f.evaluate(x, y)  # averaged accuracy over all 3 samples in x\n",
    "print(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8030937910079956\n",
      "[[-0.78104687  0.18748793  0.16346708  0.24803984  0.18205206]\n",
      " [ 0.21501946 -0.83683592  0.19003348  0.20714596  0.22463693]\n",
      " [ 0.20000091  0.23285127  0.26842937 -0.87474263  0.17346108]]\n"
     ]
    }
   ],
   "source": [
    "from singa import loss\n",
    "\n",
    "x = tensor.Tensor((3, 5))\n",
    "x.uniform(0, 1)  # randomly genearte the prediction activation\n",
    "y = tensor.from_numpy(np.array([0, 1, 3], dtype=np.int))  # set the truth\n",
    "\n",
    "f = loss.SoftmaxCrossEntropy()\n",
    "l = f.forward(True, x, y)  # l is tensor with 3 loss values\n",
    "g = f.backward()  # g is a tensor containing all gradients of x w.r.t l\n",
    "print(l.l1())\n",
    "print(tensor.to_numpy(g))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<singa.tensor.Tensor at 0x7f539260f710>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from singa import optimizer\n",
    "\n",
    "sgd = optimizer.SGD(lr=0.01, momentum=0.9, weight_decay=1e-4)\n",
    "p = tensor.Tensor((3,5))\n",
    "p.uniform(-1, 1)\n",
    "g = tensor.Tensor((3,5))\n",
    "g.gaussian(0, 0.01)\n",
    "\n",
    "sgd.apply(1, g, p, 'param')  # use the global lr=0.1 for epoch 1\n",
    "sgd.apply_with_lr(2, 0.03, g, p, 'param')  # use lr=0.03 for epoch 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FeedForwardNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('conv1', (32, 32, 32))\n",
      "('relu1', (32, 32, 32))\n",
      "('pool1', (32, 16, 16))\n",
      "('flat', (8192,))\n",
      "('dense', (10,))\n",
      "['conv1/weight', 'conv1/bias', 'dense/weight', 'dense/bias']\n"
     ]
    }
   ],
   "source": [
    "from singa import net as ffnet\n",
    "layer.engine = 'singacpp'\n",
    "net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())\n",
    "net.add(layer.Conv2D('conv1', 32, 5, 1, input_sample_shape=(3,32,32,)))\n",
    "net.add(layer.Activation('relu1'))\n",
    "net.add(layer.MaxPooling2D('pool1', 3, 2))\n",
    "net.add(layer.Flatten('flat'))\n",
    "net.add(layer.Dense('dense', 10))\n",
    "\n",
    "# init parameters\n",
    "for p in net.param_values():\n",
    "    if len(p.shape) == 0:\n",
    "        p.set_value(0)\n",
    "    else:\n",
    "        p.gaussian(0, 0.01)\n",
    "print(net.param_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('conv1', (32, 32, 32))\n",
      "('relu1', (32, 32, 32))\n",
      "('pool1', (32, 16, 16))\n",
      "('flat', (8192,))\n",
      "('dense', (10,))\n"
     ]
    }
   ],
   "source": [
    "layer.engine = 'cudnn'\n",
    "net = ffnet.FeedForwardNet(loss.SoftmaxCrossEntropy(), metric.Accuracy())\n",
    "net.add(layer.Conv2D('conv1', 32, 5, 1, input_sample_shape=(3,32,32,)))\n",
    "net.add(layer.Activation('relu1'))\n",
    "net.add(layer.MaxPooling2D('pool1', 3, 2))\n",
    "net.add(layer.Flatten('flat'))\n",
    "net.add(layer.Dense('dense', 10))\n",
    "\n",
    "# init parameters\n",
    "for p in net.param_values():\n",
    "    if len(p.shape) == 0:\n",
    "        p.set_value(0)\n",
    "    else:\n",
    "        p.gaussian(0, 0.01)\n",
    "        \n",
    "# move net onto gpu\n",
    "dev = device.create_cuda_gpu()\n",
    "net.to_device(dev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Next: [Simple models](./regression.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "py3",
   "language": "python",
   "name": "py3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}