#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

from __future__ import division

from collections import Counter, deque
import numpy as np
import math

from singa import tensor
from singa import utils
from .tensor import Tensor
from . import singa_wrap as singa

CTensor = singa.Tensor
training = False


def axis_helper(y_shape, x_shape):
    """
    check which axes the x has been broadcasted
    Args:
        y_shape: the shape of result
        x_shape: the shape of x
    Return:
        a tuple refering the axes 
    """
    res = []
    j = len(x_shape) - 1
    for i in range(len(y_shape) - 1, -1, -1):
        if j < 0 or x_shape[j] != y_shape[i]:
            res.append(i)
        j -= 1
    return tuple(res[::-1])


def back_broadcast(y_shape, x_shape, x):
    """
    for a brodcasted tensor, restore its shape of x from y_shape to x_shape
    Args:
        y_shape: the shape of result
        x_shape: the shape of x
        x: the input
    Return:
        a tensor
    """
    if y_shape != x_shape:
        x = tensor.from_raw_tensor(x)
        axis = axis_helper(y_shape, x_shape)
        x = tensor.sum(x, axis)
        x = tensor.reshape(x, x_shape)
        x = x.data
    return x


def infer_dependency(op):
    """
    Infer the dependency of all operations with the
    given op as the last operation.
    Operation A is depending on B if A uses the output(s) of B.

    Args:
        op: an Operation instance, e.g. the loss operation.

    Return:
        a Counter instance with the operation as the key,
        and the number of operations that are depending on it as the value;
        and a Counter instance with the id of the output tensor as the key, and 
        the number of operations that are depending on it as the value.
    """

    # current op is not inserted into the dependency_count
    # if the current op is not a terminal op, then this function may just
    # count dependency of a branch.
    op_count = Counter()
    tensor_count = Counter()
    queue = deque([op])
    while len(queue) > 0:
        cur_op = queue.pop()
        for src_op, xid, _, _ in cur_op.src:
            if src_op not in op_count:
                op_count[src_op] = 1
                queue.append(src_op)
            else:
                op_count[src_op] += 1
            tensor_count[xid] += 1
    return op_count, tensor_count


def gradients(y, dy=None):
    """
    Compute the gradients of the output w.r.t the parameters

    Args:
        y: the output tensor, e.g., the loss
        dy: gradient of the target w.r.t y; None indicates the gradient is 1.0;
            it can be used to rescale the loss.

    Return:
        a dictionary storing the gradient tensors of all tensors
            whose stores_grad is true (e.g. parameter tensors)
    """
    grads = {}  # mapping: x->dx if x.stores_grad
    for p, dp in backward(y, dy):
        grads[p] = dp
    return grads


def backward(y, dy=None):
    """
    Run the backward propagation starting at y.
    Args:
        y: a Tensor instance, usually the loss
        dy: a number or a Tensor instance, for the gradient of the
            objective/loss w.r.t y, usually None, i.e., 1.0
    Return:
        yeild the parameter (tensor with stores_grad true) and the
            gradient tensors.
    """
    assert isinstance(y, Tensor), "wrong input type."
    op_dep, tensor_dep = infer_dependency(y.creator)
    assert y.size() == 1, ("y must be a Tensor with a single value;"
                           "size of y is % d" % y.size())

    # by default the dy is a tensor with 1.0 for each sample;
    if dy is None:
        dy = float(1.0)
    elif isinstance(dy, Tensor):
        dy = dy.data
    else:
        dy = float(dy)

    # ready is a queue of (operation, dy list)
    ready = deque([(y.creator, (dy,))])
    not_ready = {}  # mapping: op->[dy]

    if y.stores_grad:
        # gradients[y] = dy
        if isinstance(dy, float):
            g = np.array(dy)
        else:
            g = dy
        tg = Tensor(device=g.device(), data=g)
        yield (y, tg)

    while len(ready) > 0:
        op, dys = ready.pop()
        if not op.requires_grad or isinstance(op, Dummy):
            continue
        # if not isinstance(op, tensor.Dummy):
        dxs = op._do_backward(*dys)
        # TODO src and dx must match

        assert len(op.src) == len(dxs), (
            "the number of src ops (=%d) and dx (=%d) not match" %
            (len(op.src), len(dxs)))
        for (src_op, x_id, y, y_stores_grad), dx in zip(op.src, dxs):
            # prefix x is w.r.t op; prefix y is w.r.t src_op.
            # x_id is the python id of one input arg of src_op, denoted as x.
            # y_idx (below) is the index of x among the outputs of src_op.
            # not_ready[src_op][y_idx] records the intermediate gradient
            # of the y_idx'th output of src_op. 'intermediate gradient'
            # indicates that if this output is used in multiple children
            # operations, then we have to add the graident (dx) from all these
            # children operations. When src_op is ready, it means that
            # the gradient of all its outputs are available, i.e. all children
            # operations have been backwarded.
            # y is None if y.stores_grad is false; otherwise it is a Tensor

            if isinstance(src_op, Dummy) and (not src_op.stores_grad):
                continue

            y_idx = src_op.y_id2idx[x_id]
            if src_op not in not_ready:
                # src_op may have mulitple outputs
                not_ready[src_op] = [None for _ in src_op.y_id2idx]
                not_ready[src_op][y_idx] = dx
            else:
                dxs_ = not_ready[src_op]
                if dxs_[y_idx] is None:
                    dxs_[y_idx] = dx
                else:
                    # add the gradient from another children operation that
                    # uses y_idx'th output of src_op as input arg
                    dxs_[y_idx] += dx

            op_dep[src_op] -= 1
            tensor_dep[x_id] -= 1
            if y_stores_grad and tensor_dep[x_id] == 0:
                # store the gradient for final return, e.g. for parameters.
                # it may cause a delay to yield. Only after src_op's all
                # output tensors have recieved the gradients, then output
                g = not_ready[src_op][y_idx]
                tg = Tensor(device=g.device(),
                            data=g,
                            name=src_op.grad_name(y_idx))
                yield (y, tg)

            if op_dep[src_op] == 0:
                if src_op.requires_grad is True:
                    assert not isinstance(
                        src_op, Dummy), "Dummy op does not do backward()"
                    ready.append((src_op, not_ready[src_op]))
                del not_ready[src_op]
        del op  # delete the operation to free all tensors from this op


class Operation(object):
    """
    An operation includes the forward and backward function of
    tensor calculation.
    Steps to add a specific operation Xxxx:
    1. create a subclass of Operation, name it as Xxxx
    2. override the forward() and backward(); The arguments of forward()
       and backward() should only include CTensor;
    """

    op_count = 0

    def __init__(self, name=None):
        if name is None:
            self.name = "{}#{}".format(self.__class__.__name__,
                                       Operation.op_count)
            Operation.op_count += 1
        else:
            self.name = name

    def __call__(self, *xs):
        return self._do_forward(*xs)

    def output_name(self, idx):
        """
        Args:
            idx: index of the output among all outputs

        Return:
            the name of the output tensor
        """
        return "{}:{}".format(self.name, idx)

    def grad_name(self, idx):
        """
        Args:
            idx: index of the output among all outputs

        Return:
            the name of the gradient of the output tensor
        """
        return "{}_g".format(self.output_name(idx))

    def _do_forward(self, *xs):
        """
        Do not call this function from user code. It is called by __call__().
        Args:
            xs, Tensor instance(s)
        Returns:
            Tensor instance(s)
        """
        # TODO add the pre hook
        assert all([isinstance(x, Tensor) for x in xs
                   ]), "xs should include only Tensor instances"

        # need to do backward if any of its input arg needs gradient
        self.requires_grad = any([x.requires_grad for x in xs])

        self.src = []
        for x in xs:
            if x.stores_grad:
                # store the tensor whose gradient needs be returned in
                # backward(), e.g. if x is parameter
                self.src.append((x.creator, id(x), x, x.stores_grad))
            else:
                # for intermediate tensors, they will be released soon;
                # no need to store them --> use None
                self.src.append((x.creator, id(x), None, x.stores_grad))

        # get the CTensor (data) if the input arg is Tensor
        xs = tuple(x.data for x in xs)
        ys = self.forward(*xs)
        if not isinstance(ys, tuple):
            ys = (ys,)
        # create Tensor based on CTensor(data);
        # assume outputs are all Tensor instances
        ys = tuple(
            Tensor(
                device=y.device(),
                data=y,
                requires_grad=self.requires_grad,
                creator=self,
                name=self.output_name(idx),
            ) for idx, y in enumerate(ys))
        # map from python id to output index
        self.y_id2idx = {id(y): i for i, y in enumerate(ys)}
        # TODO add the post hook
        return ys

    def _do_backward(self, *dys):
        dxs = self.backward(*dys)
        if not isinstance(dxs, tuple):
            dxs = (dxs,)
        return dxs

    def forward(self, *xs):
        """Forward propagation.
        Args:
            xs: input args consisting of only CTensors.
        Returns:
            CTensor instance(s)
        """
        raise NotImplementedError

    def backward(self, *dys):
        """ Backward propagation.
        Args:
            dys: input args consisting of only CTensors.
        Returns:
            CTensor instance(s)
        """
        raise NotImplementedError

    def get_params(self):
        return []


class Dummy(Operation):
    """Dummy operation whice serves as a placehoder for autograd
    Args:
        name(string): set it for debug
    """

    def __init__(self, tensor, name=None):
        super(Dummy, self).__init__(name)
        self.src = []
        self.y_id2idx = {id(tensor): 0}
        self.tensor = tensor
        self.requires_grad = False

    def output_name(self, idx):
        return self.name

    def grad_name(self, idx):
        return "{}_g".format(self.name)

    def __getattr__(self, name):
        return self.tensor.__getattribute__(name)


class Mean(Operation):
    """
    Element-wise mean of each of the input CTensors.
    """

    def __init__(self):
        super(Mean, self).__init__()

    def forward(self, *l):
        """
        Args:
            l (a list of CTensor): a list of CTensor for element-wise mean.
        Returns:
            a new CTensor.
        """
        if training:
            self.l = len(l)
        assert (len(l) > 0)
        x = singa.Tensor(list(l[0].shape()), l[0].device())
        x.SetFloatValue(0.0)
        for i in range(len(l)):
            x += l[i]
        return singa.MultFloat(x, 1 / len(l))

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy.
        Returns:
            a list of dx (CTensor).
        """
        return [singa.MultFloat(dy, 1 / self.l)] * self.l


def mean(*l):
    """
    Element-wise mean of each of the input tensors.
    Args:
        l (a list of Tensor): element-wise mean operator.
    Returns:
        a new Tensor.
    """
    return Mean()(*l)[0]


class ReLU(Operation):
    """
    Relu means rectified linear function, i.e, y = max(0, x) is applied to the 
    CTensor elementwise.
    """

    def __init__(self):
        super(ReLU, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): input tensor.
        Returns:
            a new CTensor whose element y = x if x >= 0; otherwise 0.
        """
        if training:
            self.input = x
        return singa.ReLU(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy.
        Returns:
            dx (CTensor): dL / dx = dy if x >= 0; otherwise 0.
        """
        return singa.ReLUBackward(dy, self.input)


def relu(x):
    """
    Relu means rectified linear function, i.e, y = max(0, x) is applied to the 
    CTensors elementwise.
    Args:
        x (Tensor): input tensor.
    Returns:
        a new Tensor whose element y = x if x >= 0; otherwise 0.
    """
    return ReLU()(x)[0]


class Less(Operation):
    """
    Returns the tensor resulted from performing the less logical operation 
    elementwise on the input CTensors x and y.
    """

    def __init__(self):
        super(Less, self).__init__()

    def forward(self, x, y):
        """
        Return a<b, where a and b are CTensor.
        """
        cur = singa.LTFloat(singa.__sub__(x, y), 0)
        if training:
            self.cache = cur
        return cur

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss.
        Raises:
            AssertionError: no backward function for this operator.
        """
        assert False, ('no backward function for less')


def less(x, y):
    """
    Return a<b, where a and b are CTensor.
    """
    return Less()(x, y)[0]


class Clip(Operation):
    """
    Clip operator limits the given input within an interval. The interval 
    is specified by the inputs 'min' and 'max'.
    """

    def __init__(self, min, max):
        """
        Args:
            min (float): min value, under which element is replaced by min.
            max (float): max value, above which element is replaced by max.
        """
        super(Clip, self).__init__()
        self.max = max
        self.min = min

    def forward(self, x):
        """
        Args:
            x (CTensor): input tensor
        Returns:
            a new CTensor with np.clip(x,min,max)
        """
        self.mask = singa.Tensor(list(x.shape()), x.device())
        self.mask.SetFloatValue(1.0)

        if self.min is not None:
            mask0 = singa.LTFloat(x, self.min)
            mask1 = singa.GEFloat(x, self.min)
            self.mask = singa.__mul__(mask1, self.mask)
            x = singa.__add__(singa.MultFloat(mask0, self.min),
                              singa.__mul__(mask1, x))

        if self.max is not None:
            mask0 = singa.GTFloat(x, self.max)
            mask1 = singa.LEFloat(x, self.max)
            self.mask = singa.__mul__(mask1, self.mask)
            x = singa.__add__(singa.MultFloat(mask0, self.max),
                              singa.__mul__(mask1, x))

        return x

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy
        Returns:
            dx (CTensor): dL / dx
        """
        return singa.__mul__(dy, self.mask)


def clip(x, min=None, max=None):
    """
    Clip operator limits the given input within an interval. The interval 
    is specified by the inputs 'min' and 'max'.
    Args:
        x (Tensor): input tensor
        min (float): Minimum value, under which element is replaced by min.
        max (float): Maximum value, above which element is replaced by max.
    Returns:
        a new Tensor with np.clip(x,min,max).
    """
    return Clip(min, max)(x)[0]


class Identity(Operation):
    """
    Init a identity operator
    """

    def __init__(self):
        super(Identity, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): input tensor.
        Returns:
            the same CTensor x.
        """
        return x

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy.
        Returns:
            dx (CTensor): dL / dx.
        """
        return dy


def identity(x):
    """
    Init a identity operator.
    Args:
        x (Tensor): input tensor.
    Returns:
        the same Tensor with x.
    """
    return Identity()(x)[0]


class Matmul(Operation):
    """
    Init matrix multiplication operator.
    """

    def __init__(self):
        super(Matmul, self).__init__()

    def forward(self, x, w):
        """
        Return `np.matmul(x,w)`, where x and w are CTensor.
        """
        if training:
            self.input = (x, w)
        return singa.Mult(x, w)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss.
        Returns:
            a tuple for (dx, dw).
        """
        return (
            singa.Mult(dy, singa.DefaultTranspose(self.input[1])),
            singa.Mult(singa.DefaultTranspose(self.input[0]), dy),
        )


def matmul(x, w):
    """
    Return `np.matmul(x,w)`, where x and w are Tensor.
    """
    return Matmul()(x, w)[0]


class Greater(Operation):
    """
    Returns the tensor resulted from performing the greater logical 
    operation elementwise on the input tensors A and B.
    """

    def __init__(self):
        super(Greater, self).__init__()

    def forward(self, x, y):
        """
        Return a>b, where a and b are CTensor.
        """
        cur = singa.GTFloat(singa.__sub__(x, y), 0)
        if training:
            self.cache = cur
        return cur

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss.
        Raises:
            AssertionError: no backward function for this operator.
        """
        assert False, ('no backward function for greater')


def greater(x, y):
    """
    Return a>b, where a and b are Tensor.
    """
    return Greater()(x, y)[0]


class AddBias(Operation):
    """
    Add Bias to each row / column of the Tensor, depending on the axis arg.
    """

    def __init__(self, axis=0):
        """
        To indicate the calculation axis, 0 for row, 1 for column.
        Args:
            axis (int): 0 or 1, default is 0.
        """
        super(AddBias, self).__init__()
        self.axis = axis

    def forward(self, x, b):
        """
        Args:
            x (CTensor): matrix.
            b (CTensor): bias to be added.
        Return:
            the result Tensor
        """
        if self.axis == 0:
            singa.AddRow(b, x)
        elif self.axis == 1:
            singa.AddColumn(b, x)
        return x

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss.
        Return:
            a tuple for (db, dx), db is data for dL / db, dx is data
            for dL / dx.
        """
        if self.axis == 0:
            return dy, singa.Sum(dy, 0)
        elif self.axis == 1:
            return dy, singa.Sum(dy, 0)


def add_bias(x, b, axis=0):
    """
    Add Bias to each row / column of the Tensor, depending on the axis arg.
    Args:
        x (Tensor): matrix.
        b (Tensor): bias to be added.
        axis (int): 0 or 1, default is 0.
    Return:
        the result Tensor
    """
    return AddBias(axis)(x, b)[0]


class Reshape(Operation):
    """
    Reshape the input tensor similar to np.reshape. 
    """

    def __init__(self, shape):
        """
        Args:
            shape (list of int): Specified shape for output. At most one
                dimension of the new shape can be -1. In this case, the 
                value is inferred from the size of the tensor and the 
                remaining dimensions. A dimension could also be 0, 
                in which case the actual dimension value is unchanged 
                (i.e. taken from the input tensor).
        """
        super(Reshape, self).__init__()
        self.shape = list(shape)

    def forward(self, x):
        """
        Args:
            x (CTensor): matrix.
        Return:
            the result CTensor
        """
        self._shape = x.shape()
        shape = self.shape
        # handle the shape with 0
        shape = [
            self._shape[i]
            if i < len(self._shape) and shape[i] == 0 else shape[i]
            for i in range(len(shape))
        ]
        # handle the shape with -1
        hidden_shape = int(np.prod(self._shape) // np.abs(np.prod(shape)))
        self.cache = [s if s != -1 else hidden_shape for s in shape]
        return singa.Reshape(x, self.cache)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy
        Returns:
            dx (CTensor): dL / dx
        """
        return singa.Reshape(dy, self._shape)


def reshape(x, shape):
    """
    Reshape the input tensor similar to mp.reshape. 
    Args:
        x (Tensor): matrix.
        shape (list of int): Specified shape for output. At most one
            dimension of the new shape can be -1. In this case, the 
            value is inferred from the size of the tensor and the 
            remaining dimensions. A dimension could also be 0, 
            in which case the actual dimension value is unchanged 
            (i.e. taken from the input tensor).
    Return:
        the result Tensor
    """
    return Reshape(shape)(x)[0]


class PRelu(Operation):
    """
    PRelu applies the function `f(x) = slope * x` for x < 0, 
    `f(x) = x` for x >= 0 to the data tensor elementwise.
    """

    def __init__(self):
        super(PRelu, self).__init__()

    def forward(self, x, slope):
        """
        Args:
            x (CTensor): matrix.
        Return:
            the result CTensor
        """
        mask0 = singa.LTFloat(x, 0.0)
        res = singa.__mul__(x, mask0)
        res = singa.__mul__(res, slope)
        res += singa.ReLU(x)
        if training:
            self.input = x
            self.slope = slope
            self.mask0 = mask0
            self.shape0 = list(x.shape())
            self.shape1 = list(slope.shape())
            self.shape3 = list(res.shape())
        return res

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy
        Returns:
            dx (CTensor): dL / dx
        """
        dx1mask = singa.GEFloat(self.input, 0.0)
        dx2 = singa.__mul__(self.mask0, self.slope)
        dx = singa.__add__(dx1mask, dx2)
        dx = singa.__mul__(dy, dx)
        dslope = singa.__mul__(dy, singa.__mul__(self.mask0, self.input))
        if (type(dy) == float) or self.shape0 == self.shape1:
            assert self.shape0 == self.shape1, ('should have same shape')
            return dx, dslope
        # handle broadcast
        dx = back_broadcast(self.shape3, self.shape0, dx)
        dslope = back_broadcast(self.shape3, self.shape1, dslope)
        return dx, dslope


def prelu(x, slope):
    """
    PRelu applies the function `f(x) = slope * x` for x < 0, 
    `f(x) = x` for x >= 0 to the data tensor elementwise.
    Args:
        x (Tensor): matrix.
    Return:
        the result Tensor
    """
    return PRelu()(x, slope)[0]


class Add(Operation):
    """
    Performs element-wise binary addition.
    """

    def __init__(self):
        super(Add, self).__init__()

    def forward(self, a, b):
        """
        Return `a+b`, where a and b are CTensor.
        """
        res = singa.__add__(a, b)
        if training:
            self.shape0 = list(a.shape())
            self.shape1 = list(b.shape())
            self.shape3 = list(res.shape())
        return res

    def backward(self, dy):
        """
        Args:
            dy(CTensor): dL / dy
        Return:
            a tuple for (dx0, dx1), dx0 is data for dL / da, dx1 is data
            for dL / db.
        """
        dx0, dx1 = dy, dy
        if (type(dy) == float) or self.shape0 == self.shape1:
            assert self.shape0 == self.shape1, ('should have same shape')
            return dx0, dx1
        # handle broadcast
        dx0 = back_broadcast(self.shape3, self.shape0, dx0)
        dx1 = back_broadcast(self.shape3, self.shape1, dx1)
        return dx0, dx1


def add(a, b):
    """
    Return `a+b`, where a and b are Tensor.
    """
    return Add()(a, b)[0]


class Elu(Operation):
    """
    `f(x) = alpha * (exp(x) - 1.)` for x < 0, `f(x) = x` for x >= 0., is applied to 
    the tensor elementwise.
    """

    def __init__(self, alpha=1.):
        """
        Args:
            alpha (float): Coefficient of ELU, default is 1.0
        """
        super(Elu, self).__init__()
        self.alpha = alpha

    def forward(self, x):
        """
        Args:
            x (CTensor): matrix
        Returns:
            a CTensor for the result
        """
        #f(x) = alpha * (exp(x) - 1.) for x < 0, f(x) = x for x >= 0
        if training:
            self.input = x
        x1 = singa.LTFloat(x, 0.0)
        x1 *= x
        x1 = singa.MultFloat(singa.SubFloat(singa.Exp(x1), 1.0), self.alpha)
        x2 = singa.ReLU(x)
        x1 += x2
        return x1

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy
        Returns:
            dx (CTensor): dL / dx
        """
        dx1mask = singa.LTFloat(self.input, 0.0)
        dx = singa.MultFloat(singa.Exp(self.input), self.alpha)
        dx *= dx1mask

        dx2mask = singa.GEFloat(self.input, 0.0)

        dx += dx2mask
        dx *= dy
        return dx


def elu(x, alpha=1):
    """
    `f(x) = alpha * (exp(x) - 1.)` for x < 0, `f(x) = x` for x >= 0., is applied to 
    the tensor elementwise.
    Args:
        x (Tensor): matrix
        alpha (float): Coefficient of ELU, default is 1.0
    Returns:
        a Tensor for the result
    """
    return Elu(alpha)(x)[0]


class Equal(Operation):
    """
    Returns the tensor resulted from performing the equal logical operation 
    elementwise on the input tensors x and y.
    """

    def __init__(self):
        super(Equal, self).__init__()

    def forward(self, x, y):
        """
        Return `a=b`, where a and b are CTensor.
        """
        m = singa.__sub__(x, y)
        cur = singa.__mul__(singa.GEFloat(m, 0), singa.LEFloat(m, 0))
        return cur

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no backward function for equal')


def equal(x, y):
    """
    Return `a=b`, where a and b are Tensor.
    """
    return Equal()(x, y)[0]


class SeLU(Operation):
    """
    `y = gamma * (alpha * e^x - alpha)` for x <= 0, `y = gamma * x` for x > 0 
    is applied to the tensor elementwise.
    """

    def __init__(self, alpha=1.67326, gamma=1.0507):
        """
        Args:
            alpha (float): Coefficient of SELU default to 1.67326
            gamma (float): Coefficient of SELU default to 1.0507
        """
        super(SeLU, self).__init__()
        self.alpha = alpha
        self.gamma = gamma

    def forward(self, x):
        """
        Args:
            x (CTensor): matrix
        Returns:
            a CTensor for the result
        """
        #y = gamma * (alpha * e^x - alpha) for x <= 0, y = gamma * x for x > 0
        if training:
            self.input = x
        x1 = singa.LEFloat(x, 0.0)
        x1 *= x
        x1 = singa.MultFloat(singa.SubFloat(singa.Exp(x1), 1.0),
                             self.alpha * self.gamma)
        x2 = singa.ReLU(x)
        x2 = singa.MultFloat(x2, self.gamma)
        x1 += x2
        return x1

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy
        Returns:
            dx (CTensor): dL / dx
        """
        dx1mask = singa.LEFloat(self.input, 0.0)
        dx1 = singa.MultFloat(singa.Exp(self.input), self.gamma * self.alpha)
        dx1 = singa.__mul__(dx1mask, dx1)

        dx2mask = singa.GTFloat(self.input, 0.0)
        dx2 = singa.MultFloat(dx2mask, self.gamma)

        dx = singa.__add__(dx1, dx2)
        dx *= dy
        return dx


def selu(x, alpha=1.67326, gamma=1.0507):
    """
    `y = gamma * (alpha * e^x - alpha)` for x <= 0, `y = gamma * x` for x > 0 
    is applied to the tensor elementwise.
    Args:
        x (Tensor): matrix
        alpha (float): Coefficient of SELU default to 1.67326
        gamma (float): Coefficient of SELU default to 1.0507
    Returns:
        a Tensor for the result
    """
    return SeLU(alpha, gamma)(x)[0]


class SoftMax(Operation):
    """
    Apply SoftMax for each row of the Tensor or each column of the Tensor
    according to the parameter axis.
    """

    def __init__(self, axis=1):
        """
        Args:
            axis (int): axis of softmax, default to 1
        """
        super(SoftMax, self).__init__()
        self.axis = axis

    def forward(self, x):
        """
        Args:
            x (CTensor): the input 1d or 2d tensor
        Returns:
            the result CTensor
        """
        self.output = singa.SoftMax(x, self.axis)
        return self.output

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy
        Returns:
            dx (CTensor): dL / dx
        """
        return singa.SoftMaxBackward(dy, self.axis, self.output)


def softmax(x, axis=1):
    """
    Apply SoftMax for each row of the Tensor or each column of the Tensor
    according to the parameter axis.
    Args:
        x (Tensor): the input 1d or 2d tensor
        axis (int): axis of softmax, default to 1
    Returns:
        the result Tensor
    """
    return SoftMax(axis)(x)[0]


class Sum(Operation):
    """
    Element-wise sum of each of the input tensors
    """

    def __init__(self):
        super(Sum, self).__init__()

    def forward(self, *l):
        """
        Args:
            l (a list of CTensor): element-wise sum operator
        Returns:
            a CTensor for the result
        """
        if training:
            self.l = len(l)
        assert (len(l) > 0)
        x = singa.Tensor(list(l[0].shape()), l[0].device())
        x.SetFloatValue(0.0)
        for i in range(len(l)):
            x += l[i]
        return x

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy
        Returns:
            dx (CTensor): dL / dx
        """
        return [dy] * self.l


def sum(*l):
    """
    Element-wise sum of each of the input tensors
    Args:
        l (a list of Tensor): element-wise sum operator
    Returns:
        a Tensor for the result
    """
    return Sum()(*l)[0]


class CrossEntropy(Operation):

    def __init__(self):
        super(CrossEntropy, self).__init__()

    """
    Calculte negative log likelihood loss for a batch of training data.
    """

    def forward(self, x, t):
        """
        Args:
            x (CTensor): 1d or 2d tensor, the prediction data(output)
                         of current network.
            t (CTensor): 1d or 2d tensor, the target data for training.
        Returns:
            loss (CTensor): scalar.
        """
        loss = singa.SumAll(singa.__mul__(t, singa.Log(x)))
        loss /= -x.shape()[0]
        self.x = x
        self.t = t
        self.input = (x, t)
        return loss

    def backward(self, dy=1.0):
        """
        Args:
            dy (float or CTensor): scalar, accumulate gradient from outside
                                of current network, usually equal to 1.0
        Returns:
            dx (CTensor): data for the dL /dx, L is the loss, x is the output
                          of current network. note that this is true for
                          dy = 1.0
        """
        dx = singa.__div__(self.t, self.x)
        dx *= float(-1.0 / self.x.shape()[0])
        if isinstance(dy, float):
            # dtype of dy: float
            dx *= dy
            return dx, None
        elif isinstance(dy, CTensor):
            pass  # TODO, broadcast elementwise multiply seems not support


def cross_entropy(y, t):
    return CrossEntropy()(y, t)[0]


class SoftMaxCrossEntropy(Operation):

    def __init__(self, t):
        super(SoftMaxCrossEntropy, self).__init__()
        self.t = t.data

    def forward(self, x):
        self.p = singa.SoftMax(x)
        ret = singa.CrossEntropyFwd(self.p, self.t)
        loss = singa.SumAll(ret)
        loss /= x.shape()[0]
        return loss

    def backward(self, dy=1.0):
        dx = singa.SoftmaxCrossEntropyBwd(self.p, self.t)
        dx /= float(self.p.shape()[0])
        return dx


def softmax_cross_entropy(x, t):
    # x is the logits and t is the ground truth; both are 2D.
    return SoftMaxCrossEntropy(t)(x)[0]


class MeanSquareError(Operation):

    def __init__(self):
        super(MeanSquareError, self).__init__()

    def forward(self, x, t):
        self.err = singa.__sub__(x, t)
        sqr = singa.Square(self.err)
        loss = singa.SumAll(sqr)
        loss /= (x.shape()[0] * 2)
        return loss

    def backward(self, dy=1.0):
        dx = self.err
        dx *= float(1 / self.err.shape()[0])
        dx *= dy
        return dx, None


def mse_loss(x, t):
    return MeanSquareError()(x, t)[0]


def ctensor2numpy(x):
    """
    To be used in SoftMax Operation.
    Convert a singa_tensor to numpy_tensor.
    """
    np_array = x.GetFloatValue(int(x.Size()))
    return np_array.reshape(x.shape())


class Flatten(Operation):
    """
    Flattens the input tensor into a 2D matrix. If input tensor has shape 
    `(d_0, d_1, ... d_n)` then the output will have shape `(d_0 X d_1 ... 
    d_(axis-1), d_axis X d_(axis+1) ... X dn)`.
    """

    def __init__(self, axis=1):
        """
        Args:
            axis (int): Indicate up to which input dimensions (exclusive) 
                should be flattened to the outer dimension of the output. The 
                value for axis must be in the range [-r, r], where r is the 
                rank of the input tensor. Negative value means counting 
                dimensions from the back. When axis = 0, the shape of the 
                output tensor is `(1, (d_0 X d_1 ... d_n)`, where the shape 
                of the input tensor is `(d_0, d_1, ... d_n)`.
        Returns:
            the result CTensor
        """
        super(Flatten, self).__init__()
        self.axis = axis

    def forward(self, x):
        """
        Args:
            x (CTensor): the input tensor
        Returns:
            the result CTensor
        """
        self.shape = list(x.shape())
        shape, axis = self.shape, self.axis
        # the axis must be within this range (0, r-1)
        assert axis <= len(
            shape) - 1 or axis >= 0, "the axis must be within (0, %d-1)" % len(
                shape)
        # calculate the new shape
        new_shape = (1, int(np.prod(shape))) if axis == 0 else (
            int(np.prod(shape[0:axis]).astype(int)),
            int(np.prod(shape[axis:]).astype(int)))
        y = singa.Reshape(x, new_shape)
        return y

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss
        Returns:
            dx (CTensor): data for the dL / dx, L is the loss,
        """
        dx = singa.Reshape(dy, self.shape)
        return dx


def flatten(x, axis=1):
    """
    Flattens the input tensor into a 2D matrix. If input tensor has shape 
    `(d_0, d_1, ... d_n)` then the output will have shape `(d_0 X d_1 ... 
    d_(axis-1), d_axis X d_(axis+1) ... X dn)`.
    Args:
        x (Tensor): the input tensor
        axis (int): Indicate up to which input dimensions (exclusive) 
            should be flattened to the outer dimension of the output. The 
            value for axis must be in the range [-r, r], where r is the 
            rank of the input tensor. Negative value means counting 
            dimensions from the back. When axis = 0, the shape of the 
            output tensor is `(1, (d_0 X d_1 ... d_n)`, where the shape 
            of the input tensor is `(d_0, d_1, ... d_n)`.
    Returns:
        the result Tensor
    """
    return Flatten(axis)(x)[0]


class Layer(object):

    def __init__(self):
        self.allow_params = []
        pass

    def device_check(self, *inputs):
        x_device = inputs[0].device
        x_dev_id = x_device.id()
        for var in inputs:
            if var.device.id() != x_dev_id:
                var.to_device(x_device)

    def find_sublayers(self):
        # return a list whose elements are in form of (attribute_name,
        # sublayer)
        sublayers = []
        for attr in self.__dict__:
            if isinstance(self.__dict__[attr], Layer):
                sublayers.append((attr, self.__dict__[attr]))
        return sublayers

    def get_params(self):
        sublayers = self.find_sublayers()
        params = dict()
        for sublayer_name, sublayer in sublayers:
            params[sublayer_name] = sublayer.get_params()
        return params

    def set_params(self, **parameters):
        # set parameters for Layer
        # input should be either a PyTensor or numpy ndarray.
        # examples: Layer.set_params(W=np.ones((in, out), dtype=np.float32)),
        # Layer.set_params(**{'block1':{'linear1':{'W':np.ones((in, out),
        # dtype=np.float32)}}})
        for (parameter_name, parameter_value) in parameters.items():
            # assert isinstance(self.__dict__[parameter_name], Layer)
            assert (parameter_name in self.__dict__
                   ), "please input correct parameters."
            if isinstance(self.__dict__[parameter_name], Layer):
                self.__dict__[parameter_name].set_params(
                    **parameters[parameter_name])
            elif isinstance(self.__dict__[parameter_name], Tensor):
                self.set_one_param(parameter_name, parameter_value)
            else:
                raise ValueError("please input correct parameters.")

    def set_one_param(self, parameter_name, parameter_value):
        assert (parameter_name in self.allow_params
               ), "please input allowed parameters."
        assert (parameter_value.shape == self.__dict__[parameter_name].shape
               ), "Shape dismatched."
        if isinstance(parameter_value, Tensor):
            self.__dict__[parameter_name].reset_like(parameter_value)
        elif isinstance(parameter_value, np.ndarray):
            self.__dict__[parameter_name].copy_from_numpy(parameter_value)
        else:
            raise ValueError("parameters should be Tensor or Numpy array.")


class Linear(Layer):
    """
    Generate a Linear operator
    """

    def __init__(self, in_features, out_features, bias=True):
        """
        Args:
            in_channels: int, the channel of input
            out_channels: int, the channel of output, also is the number of 
                filters
            bias: bool
        """
        w_shape = (in_features, out_features)
        b_shape = (out_features,)
        self.bias = bias

        self.W = Tensor(shape=w_shape, requires_grad=True, stores_grad=True)
        std = math.sqrt(2.0 / (in_features + out_features))
        self.W.gaussian(0.0, std)

        if self.bias:
            self.b = Tensor(shape=b_shape, requires_grad=True, stores_grad=True)
            self.b.set_value(0.0)

    def __call__(self, x):
        if self.bias:
            self.device_check(x, self.W, self.b)
        else:
            self.device_check(x, self.W)
        assert x.shape[1] == self.W.shape[0], (
            "Linear layer expects input features size %d received %d" %
            (self.W.shape[0], x.shape[1]))
        y = matmul(x, self.W)
        if self.bias:
            y = add_bias(y, self.b, axis=0)
        return y

    def get_params(self):
        if self.bias:
            return {"W": self.W, "b": self.b}
        else:
            return {"W": self.W}

    def set_params(self, **parameters):
        # TODO(wangwei) remove this funciton as Opeation's set_params() enough
        # set parameters for Linear Layer
        # input should be either a PyTensor or numpy ndarray.
        # examples: Linear.set_params(W=np.ones((in, out), dtype=np.float32)),
        # Linear.set_params(**{'W':np.ones((in, out), dtype=np.float32)})
        self.allow_params = ["W", "b"]
        super(Linear, self).set_params(**parameters)
        for parameter_name in parameters:
            if parameter_name is "b":
                self.bias = True


class Concat(Operation):
    """
    Concatenate a list of tensors into a single tensor. All input tensors must 
    have the same shape, except for the dimension size of the axis to 
    concatenate on.
    """

    def __init__(self, axis=0):
        """
        Args:
            axis (int): Which axis to concat on. A negative value means 
                counting dimensions from the back. Accepted range is [-r, r-1] 
                where r = rank(inputs).
        Returns:
            the result CTensor
        """
        super(Concat, self).__init__()
        self.axis = axis

    def forward(self, *xs):
        """
        Args:
            xs (a list of CTensor): List of tensors for concatenation
        Returns:
            a CTensor for the result
        """
        if training:
            offset = 0
            self.slice_point = []
            for t in xs:
                offset += t.shape()[self.axis]
                self.slice_point.append(offset)
        x = singa.VecTensor(list(xs))
        return singa.ConcatOn(x, self.axis)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss
        Returns:
            dxs (a tuple of CTensor): data for the dL / dxs, L is the loss,
        """
        assert hasattr(
            self, "slice_point"), "Please set training as True before do BP. "
        assert self.slice_point[-1] == dy.shape()[self.axis], "Shape mismatch."
        dxs = []
        last_offset = 0
        for p in self.slice_point:
            dxs.append(singa.SliceOn(dy, last_offset, p, self.axis))
            last_offset = p
        return tuple(dxs)


def cat(xs, axis=0):
    """
    Concatenate a list of tensors into a single tensor. All input tensors must 
    have the same shape, except for the dimension size of the axis to 
    concatenate on.
    Args:
        xs (a list of Tensor): List of tensors for concatenation
        axis (int): Which axis to concat on. A negative value means 
            counting dimensions from the back. Accepted range is [-r, r-1] 
            where r = rank(inputs).
    Returns:
        a Tensor for the result
    """
    return Concat(axis)(*xs)[0]


class _Conv2d(Operation):
    """
    Init a conv 2d operator
    """

    def __init__(self, handle, odd_padding=(0, 0, 0, 0)):
        """
        Args:
            handle (object): ConvHandle for cpu or CudnnConvHandle for gpu
            odd_padding (tuple of four ints):, the odd paddding is the value 
                that cannot be handled by the tuple padding (w, h) mode so 
                we need to firstly handle the input, then use the nomal padding 
                method.
        """
        super(_Conv2d, self).__init__()
        self.handle = handle
        self.odd_padding = odd_padding
        if self.odd_padding != (0, 0, 0, 0):
            self.re_new_handle = True

    def forward(self, x, W, b=None):
        """
        Args:
            x (CTensor): input
            W (CTensor): weight
            b (CTensor): bias
        Returns:
            CTensor 
        """
        assert x.nDim() == 4, "The dimensions of input should be 4D."
        if self.odd_padding != (0, 0, 0, 0):
            x = utils.handle_odd_pad_fwd(x, self.odd_padding)
            # re-new a handle with updated x
            if self.re_new_handle:
                self.re_new_handle = False
                self.handle = utils.re_new_handle(self.handle, x)

        if training:
            if self.handle.bias_term:
                self.inputs = (x, W, b)
            else:
                self.inputs = (x, W)

        if not self.handle.bias_term:
            # create empty bias tensor for Cpp API
            b = CTensor((self.handle.num_filters,), x.device())
            b.SetFloatValue(0.0)

        if (type(self.handle) != singa.ConvHandle):
            return singa.GpuConvForward(x, W, b, self.handle)
        else:
            return singa.CpuConvForward(x, W, b, self.handle)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): dL / dy
        Returns:
            dx (CTensor): dL / dx
        """
        assert training is True and hasattr(
            self, "inputs"), "Please set training as True before do BP. "

        if (type(self.handle) != singa.ConvHandle):
            dx = singa.GpuConvBackwardx(dy, self.inputs[1], self.inputs[0],
                                        self.handle)
            dW = singa.GpuConvBackwardW(dy, self.inputs[0], self.inputs[1],
                                        self.handle)
            db = singa.GpuConvBackwardb(
                dy, self.inputs[2],
                self.handle) if self.handle.bias_term else None
        else:
            dx = singa.CpuConvBackwardx(dy, self.inputs[1], self.inputs[0],
                                        self.handle)
            dW = singa.CpuConvBackwardW(dy, self.inputs[0], self.inputs[1],
                                        self.handle)
            db = singa.CpuConvBackwardb(
                dy, self.inputs[2],
                self.handle) if self.handle.bias_term else None
        if self.odd_padding != (0, 0, 0, 0):
            dx = utils.handle_odd_pad_bwd(dx, self.odd_padding)

        if db:
            return dx, dW, db

        else:
            return dx, dW


def conv2d(handle, x, W, b=None, odd_padding=(0, 0, 0, 0)):
    """
    Conv 2d operator
    Args:
        handle (object): ConvHandle for cpu or CudnnConvHandle for gpu
        x (Tensor): input
        W (Tensor): weight
        b (Tensor): bias
        odd_padding (tuple of four ints):, the odd paddding is the value 
            that cannot be handled by the tuple padding (w, h) mode so 
            we need to firstly handle the input, then use the nomal padding 
            method.
    """
    if b is None:
        return _Conv2d(handle, odd_padding)(x, W)[0]
    else:
        return _Conv2d(handle, odd_padding)(x, W, b)[0]


class Conv2d(Layer):
    """
    Generate a Conv 2d operator
    """

    def __init__(self,
                 in_channels,
                 out_channels,
                 kernel_size,
                 stride=1,
                 padding=0,
                 dilation=1,
                 group=1,
                 bias=True,
                 pad_mode="NOTSET",
                 **kwargs):
        """
        Args:
            in_channels (int): the channel of input
            out_channels (int): the channel of output, also is the number of filters
            kernel_size (int or tuple): kernel size for two direction of each 
                axis. For example, (2, 3), the first 2 means will add 2 at the 
                beginning and also 2 at the end for its axis.and if a int is 
                accepted, the kernel size will be initiated as (int, int)
            stride (int or tuple): stride, the logic is the same as kernel size.
            padding (int): tuple, list or None, padding, the logic is the same 
                as kernel size. However, if you set pad_mode as "SAME_UPPER" or 
                "SAME_LOWER" mode, you can set padding as None, and the padding 
                will be computed automatically.
            dilation (int): only support 1
            group (int): group
            bias (bool): bias
            pad_mode (string): can be NOTSET, SAME_UPPER, or SAME_LOWER, where 
                default value is NOTSET, which means explicit padding is used. 
                SAME_UPPER or SAME_LOWER mean pad the input so that the output 
                spatial size match the input. In case of odd number add the extra 
                padding at the end for SAME_UPPER and at the beginning for SAME_LOWER.
        """
        self.in_channels = in_channels
        self.out_channels = out_channels

        self.group = group

        assert (self.group >= 1 and self.in_channels %
                self.group == 0), "please set reasonable group."

        assert (self.out_channels >= self.group and self.out_channels %
                self.group == 0), "out_channels and group dismatched."

        if isinstance(kernel_size, int):
            self.kernel_size = (kernel_size, kernel_size)
        elif isinstance(kernel_size, tuple):
            self.kernel_size = kernel_size
        else:
            raise TypeError("Wrong kernel_size type.")

        if isinstance(stride, int):
            self.stride = (stride, stride)
        elif isinstance(stride, tuple):
            self.stride = stride
        else:
            raise TypeError("Wrong stride type.")

        self.odd_padding = (0, 0, 0, 0)
        if isinstance(padding, int):
            self.padding = (padding, padding)
        elif isinstance(padding, tuple) or isinstance(padding, list):
            if len(padding) == 2:
                self.padding = padding
            elif len(padding) == 4:
                _h_mask = padding[0] - padding[1]
                _w_mask = padding[2] - padding[3]
                # the odd paddding is the value that cannot be handled by the tuple padding (w, h) mode
                # so we need to firstly handle the input, then use the nomal padding method.
                self.odd_padding = (max(_h_mask, 0), max(-_h_mask, 0),
                                    max(_w_mask, 0), max(-_w_mask, 0))
                self.padding = (
                    padding[0] - self.odd_padding[0],
                    padding[2] - self.odd_padding[2],
                )
            else:
                raise TypeError("Wrong padding value.")

        if dilation != 1:
            raise ValueError("Not implemented yet")

        self.bias = bias

        self.inner_params = {
            "cudnn_prefer": "fastest",
            "workspace_MB_limit": 1024,
        }
        # TODO valid value of inner_params check

        for kwarg in kwargs:
            if kwarg not in self.inner_params:
                raise TypeError("Keyword argument not understood:", kwarg)
            else:
                self.inner_params[kwarg] = kwargs[kwarg]

        w_shape = (
            self.out_channels,
            int(self.in_channels / self.group),
            self.kernel_size[0],
            self.kernel_size[1],
        )

        self.W = Tensor(shape=w_shape, requires_grad=True, stores_grad=True)
        # std = math.sqrt(
        # 2.0 / (self.in_channels * self.kernel_size[0] * self.kernel_size[1] +
        # self.out_channels))
        std = math.sqrt(
            2.0 / (w_shape[1] * self.kernel_size[0] * self.kernel_size[1] +
                   self.out_channels))
        self.W.gaussian(0.0, std)

        if self.bias:
            b_shape = (self.out_channels,)
            self.b = Tensor(shape=b_shape, requires_grad=True, stores_grad=True)
            self.b.set_value(0.0)
        else:
            # to keep consistency when to do forward.
            self.b = None
            # Tensor(data=CTensor([]), requires_grad=False, stores_grad=False)
        self.pad_mode = pad_mode

    def __call__(self, x):
        assert x.shape[1] == self.in_channels, "in_channels mismatched"

        # if same pad mode, re-compute the padding
        if self.pad_mode in ("SAME_UPPER", "SAME_LOWER"):
            self.padding, self.odd_padding = utils.get_padding_shape(
                self.pad_mode, x.shape[2:], self.kernel_size, self.stride)

        if self.bias:
            self.device_check(x, self.W, self.b)
        else:
            self.device_check(x, self.W)

        if x.device.id() == -1:
            if self.group != 1:
                raise ValueError("Not implemented yet")
            else:
                if (not hasattr(self, "handle")) or (x.shape[0] !=
                                                     self.handle.batchsize):
                    self.handle = singa.ConvHandle(
                        x.data,
                        self.kernel_size,
                        self.stride,
                        self.padding,
                        self.in_channels,
                        self.out_channels,
                        self.bias,
                        self.group,
                    )
        else:
            if (not hasattr(self,
                            "handle")) or (x.shape[0] != self.handle.batchsize):
                self.handle = singa.CudnnConvHandle(
                    x.data,
                    self.kernel_size,
                    self.stride,
                    self.padding,
                    self.in_channels,
                    self.out_channels,
                    self.bias,
                    self.group,
                )

        y = conv2d(self.handle, x, self.W, self.b, self.odd_padding)
        return y

    def get_params(self):
        if self.bias:
            return {"W": self.W, "b": self.b}
        else:
            return {"W": self.W}

    def set_params(self, **parameters):
        # TODO(wangwei) remove it as Operation's set_params() is enough
        # input should be either a PyTensor or numpy ndarray.
        # Conv2d.set_params(W=np.ones((n, c, h, w), dtype=np.float32)),
        # Conv2d.set_params(**{'W':np.ones((n, c, h, w), dtype=np.float32)})
        self.allow_params = ["W", "b"]
        super(Conv2d, self).set_params(**parameters)
        for parameter_name in parameters:
            if parameter_name is "b":
                self.bias = True


class SeparableConv2d(Layer):
    """
    Generate a Conv 2d operator
    """

    def __init__(
            self,
            in_channels,
            out_channels,
            kernel_size,
            stride=1,
            padding=0,
            bias=False,
    ):
        """
        Args:
            in_channels (int): the channel of input
            out_channels (int): the channel of output, also is the number of filters
            kernel_size (int or tuple): kernel size for two direction of each 
                axis. For example, (2, 3), the first 2 means will add 2 at the 
                beginning and also 2 at the end for its axis.and if a int is 
                accepted, the kernel size will be initiated as (int, int)
            stride (int or tuple): stride, the logic is the same as kernel size.
            padding (int): tuple, list or None, padding, the logic is the same 
                as kernel size. However, if you set pad_mode as "SAME_UPPER" or 
                "SAME_LOWER" mode, you can set padding as None, and the padding 
                will be computed automatically.
            bias (bool): bias
        """
        self.depthwise_conv = Conv2d(
            in_channels,
            in_channels,
            kernel_size,
            stride,
            padding,
            group=in_channels,
            bias=bias,
        )

        self.point_conv = Conv2d(in_channels, out_channels, 1, bias=bias)

    def __call__(self, x):
        y = self.depthwise_conv(x)
        y = self.point_conv(y)
        return y


class BatchNorm2d(Layer):
    """
    Generate a BatchNorm 2d operator
    """

    def __init__(self, num_features, momentum=0.9):
        """
        Args:
            num_features (int): int, the channel of input
            momentum (float): Factor used in computing the running mean and 
                variance.
        """
        self.channels = num_features
        self.momentum = momentum

        param_shape = (self.channels,)

        self.scale = Tensor(shape=param_shape,
                            requires_grad=True,
                            stores_grad=True)
        self.scale.set_value(1.0)

        self.bias = Tensor(shape=param_shape,
                           requires_grad=True,
                           stores_grad=True)
        self.bias.set_value(0.0)

        self.running_mean = Tensor(shape=param_shape,
                                   requires_grad=False,
                                   stores_grad=False)
        self.running_mean.set_value(0.0)

        self.running_var = Tensor(shape=param_shape,
                                  requires_grad=False,
                                  stores_grad=False)
        self.running_var.set_value(1.0)

    def __call__(self, x):
        assert x.shape[1] == self.channels, (
            "number of channels dismatched. %d vs %d" %
            (x.shape[1], self.channels))

        self.device_check(x, self.scale, self.bias, self.running_mean,
                          self.running_var)

        if x.device.id() == -1:
            if not hasattr(self, "handle"):
                self.handle = singa.BatchNormHandle(self.momentum, x.data)
            elif x.shape[0] != self.handle.batchsize:
                self.handle = singa.BatchNormHandle(self.momentum, x.data)
        else:
            if not hasattr(self, "handle"):
                self.handle = singa.CudnnBatchNormHandle(self.momentum, x.data)
            elif x.shape[0] != self.handle.batchsize:
                self.handle = singa.CudnnBatchNormHandle(self.momentum, x.data)

        y = batchnorm_2d(
            self.handle,
            x,
            self.scale,
            self.bias,
            self.running_mean,
            self.running_var,
        )
        return y

    def get_params(self):
        return {"scale": self.scale, "bias": self.bias}

    def set_params(self, **parameters):
        # set parameters for BatchNorm2d Layer
        # input should be either a PyTensor or numpy ndarray.
        # examples:
        #   Batchnorm2d.set_params(scale=np.ones((1,), dtype=np.float32)),
        #   Batchnorm2d.set_params(**{'bias':np.ones((1), dtype=np.float32)})
        self.allow_params = ["scale", "bias"]
        super(BatchNorm2d, self).set_params(**parameters)


class _BatchNorm2d(Operation):
    """
    Carries out batch normalization as described in the paper 
    https://arxiv.org/abs/1502.03167. 
    """

    def __init__(self, handle, running_mean, running_var, name=None):
        """
        Args:
            handle (object): BatchNormHandle for cpu and CudnnBatchNormHandle 
                for gpu
            running_mean (float): the running_mean
            running_var (float): the running_var
            name (string): the name assigned to this operator
        """
        super(_BatchNorm2d, self).__init__(name)
        self.handle = handle
        self.running_mean = running_mean.data
        self.running_var = running_var.data

    def forward(self, x, scale, bias):
        """
        Args:
            x (CTensor): the input tensor
            scale (CTensor): the bias tensor
            bias (CTensor): the bias tensor
        Returns:
            the result CTensor
        """
        if training:
            if (type(self.handle) == singa.BatchNormHandle):
                y, mean, var = singa.CpuBatchNormForwardTraining(
                    self.handle, x, scale, bias, self.running_mean,
                    self.running_var)

                self.cache = (x, scale, mean, var, y, bias)
            else:
                y, mean, var = singa.GpuBatchNormForwardTraining(
                    self.handle, x, scale, bias, self.running_mean,
                    self.running_var)

                self.cache = (x, scale, mean, var)

        else:

            if (type(self.handle) == singa.BatchNormHandle):
                y = singa.CpuBatchNormForwardInference(
                    self.handle,
                    x,
                    scale,
                    bias,
                    self.running_mean,
                    self.running_var,
                )
            else:
                y = singa.GpuBatchNormForwardInference(
                    self.handle,
                    x,
                    scale,
                    bias,
                    self.running_mean,
                    self.running_var,
                )
        return y

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss
        Returns:
            dx (CTensor): data for the dL / dx, L is the loss
            ds (CTensor): data for the dL / ds, L is the loss
            db (CTensor): data for the dL / db, L is the loss
        """
        assert training is True and hasattr(
            self, "cache"), "Please set training as True before do BP. "

        if (type(self.handle) == singa.BatchNormHandle):
            x, scale, mean, var, y, bias = self.cache
            dx, ds, db = singa.CpuBatchNormBackwardx(self.handle, y, dy, x,
                                                     scale, bias, mean, var)
        else:
            x, scale, mean, var = self.cache
            dx, ds, db = singa.GpuBatchNormBackward(self.handle, dy, x, scale,
                                                    mean, var)

        return dx, ds, db


def batchnorm_2d(handle, x, scale, bias, running_mean, running_var):
    """
    Carries out batch normalization as described in the paper 
    https://arxiv.org/abs/1502.03167. 
    Args:
        handle (object): BatchNormHandle for cpu and CudnnBatchNormHandle 
            for gpu
        x (Tensor): the input tensor
        scale (Tensor): the bias tensor
        bias (Tensor): the bias tensor
        running_mean (float): the running_mean
        running_var (float): the running_var
    Returns:
        the result Tensor
    """
    return _BatchNorm2d(handle, running_mean, running_var)(x, scale, bias)[0]


class _Pooling2d(Operation):
    """
    Init a pool 2d operator
    """

    def __init__(self, handle, odd_padding=(0, 0, 0, 0)):
        """
        Args:
            handle (object): PoolingHandle for cpu or CudnnPoolingHandle for 
                gpu
            odd_padding (tuple of four int): the odd paddding is the value 
                that cannot be handled by the tuple padding (w, h) mode so 
                it needs to firstly handle the input, then use the normal 
                padding method.
        """
        super(_Pooling2d, self).__init__()
        self.handle = handle
        self.odd_padding = odd_padding
        if self.odd_padding != (0, 0, 0, 0):
            self.re_new_handle = True

    def forward(self, x):
        """
        Args:
            x (CTensor): the input tensor
        Returns:
            the result CTensor
        """
        assert x.nDim() == 4, "The dimensions of input should be 4D."
        if self.odd_padding != (0, 0, 0, 0):
            x = utils.handle_odd_pad_fwd(x, self.odd_padding)
            # re-new a handle with updated x
            if self.re_new_handle:
                self.re_new_handle = False
                self.handle = utils.re_new_handle(self.handle, x, True)

        if (type(self.handle) != singa.PoolingHandle):
            y = singa.GpuPoolingForward(self.handle, x)
        else:
            y = singa.CpuPoolingForward(self.handle, x)
        if training:
            self.cache = (x, y)
        return y

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the dL / dy, L is the loss
        Returns:
            dx (CTensor): data for the dL / dx, L is the loss,
        """
        if (type(self.handle) != singa.PoolingHandle):
            dx = singa.GpuPoolingBackward(self.handle, dy, self.cache[0],
                                          self.cache[1])
        else:
            dx = singa.CpuPoolingBackward(self.handle, dy, self.cache[0],
                                          self.cache[1])
        if self.odd_padding != (0, 0, 0, 0):
            dx = utils.handle_odd_pad_bwd(dx, self.odd_padding)

        return dx


def pooling_2d(handle, x, odd_padding=(0, 0, 0, 0)):
    """
    Pooling 2d operator
    Args:
        handle (object): PoolingHandle for cpu or CudnnPoolingHandle for 
            gpu
        x (Tensor): input
        odd_padding (tuple of four int): the odd paddding is the value 
            that cannot be handled by the tuple padding (w, h) mode so 
            it needs to firstly handle the input, then use the normal 
            padding method.
    Returns:
        the result Tensor
    """
    return _Pooling2d(handle, odd_padding)(x)[0]


class Pooling2d(Layer):
    """
    Generate a Pooling 2d operator
    """

    def __init__(self,
                 kernel_size,
                 stride=None,
                 padding=0,
                 is_max=True,
                 pad_mode="NOTSET"):
        """
        Args:
            kernel_size (int or tuple): kernel size for two direction of each 
                axis. For example, (2, 3), the first 2 means will add 2 at the 
                beginning and also 2 at the end for its axis.and if a int is 
                accepted, the kernel size will be initiated as (int, int)
            stride (int or tuple): stride, the logic is the same as kernel size.
            padding (int): tuple, list or None, padding, the logic is the same 
                as kernel size. However, if you set pad_mode as "SAME_UPPER" or 
                "SAME_LOWER" mode, you can set padding as None, and the padding 
                will be computed automatically.
            is_max (bool): is max pooling or avg pooling
            pad_mode (string): can be NOTSET, SAME_UPPER, or SAME_LOWER, where 
                default value is NOTSET, which means explicit padding is used. 
                SAME_UPPER or SAME_LOWER mean pad the input so that the output 
                spatial size match the input. In case of odd number add the extra 
                padding at the end for SAME_UPPER and at the beginning for SAME_LOWER.
        """
        if isinstance(kernel_size, int):
            self.kernel_size = (kernel_size, kernel_size)
        elif isinstance(kernel_size, tuple):
            self.kernel_size = kernel_size
        else:
            raise TypeError("Wrong kernel_size type.")

        if stride is None:
            self.stride = self.kernel_size
        elif isinstance(stride, int):
            self.stride = (stride, stride)
        elif isinstance(stride, tuple):
            self.stride = stride
            assert stride[0] > 0 or (kernel_size[0] == 1 and padding[0] == 0), (
                "stride[0]=0, but kernel_size[0]=%d, padding[0]=%d" %
                (kernel_size[0], padding[0]))
        else:
            raise TypeError("Wrong stride type.")

        self.odd_padding = (0, 0, 0, 0)
        if isinstance(padding, int):
            self.padding = (padding, padding)
        elif isinstance(padding, tuple) or isinstance(padding, list):
            if len(padding) == 2:
                self.padding = padding
            elif len(padding) == 4:
                _h_mask = padding[0] - padding[1]
                _w_mask = padding[2] - padding[3]
                # the odd paddding is the value that cannot be handled by the tuple padding (w, h) mode
                # so we need to firstly handle the input, then use the nomal padding method.
                self.odd_padding = (max(_h_mask, 0), max(-_h_mask, 0),
                                    max(_w_mask, 0), max(-_w_mask, 0))
                self.padding = (
                    padding[0] - self.odd_padding[0],
                    padding[2] - self.odd_padding[2],
                )
            else:
                raise TypeError("Wrong padding value.")

        self.is_max = is_max
        self.pad_mode = pad_mode

    def __call__(self, x):
        # if same pad mode, re-compute the padding
        if self.pad_mode in ("SAME_UPPER", "SAME_LOWER"):
            self.padding, self.odd_padding = utils.get_padding_shape(
                self.pad_mode, x.shape[2:], self.kernel_size, self.stride)

        out_shape_h = (int(
            (x.shape[2] + 2 * self.padding[0] - self.kernel_size[0]) //
            self.stride[0]) + 1)
        out_shape_w = (int(
            (x.shape[3] + 2 * self.padding[1] - self.kernel_size[1]) //
            self.stride[1]) + 1)
        if x.device.id() == -1:
            if not hasattr(self, "handle"):
                self.handle = singa.PoolingHandle(
                    x.data,
                    self.kernel_size,
                    self.stride,
                    self.padding,
                    self.is_max,
                )
            elif (x.shape[0] != self.handle.batchsize or
                  out_shape_h != self.handle.pooled_height or
                  out_shape_w != self.handle.pooled_width):
                self.handle = singa.PoolingHandle(
                    x.data,
                    self.kernel_size,
                    self.stride,
                    self.padding,
                    self.is_max,
                )
        else:
            if not hasattr(self, "handle"):
                self.handle = singa.CudnnPoolingHandle(
                    x.data,
                    self.kernel_size,
                    self.stride,
                    self.padding,
                    self.is_max,
                )
            elif (x.shape[0] != self.handle.batchsize or
                  out_shape_h != self.handle.pooled_height or
                  out_shape_w != self.handle.pooled_width):
                self.handle = singa.CudnnPoolingHandle(
                    x.data,
                    self.kernel_size,
                    self.stride,
                    self.padding,
                    self.is_max,
                )

        y = pooling_2d(self.handle, x, self.odd_padding)
        return y


class MaxPool2d(Pooling2d):
    """
    Generate a Max Pooling 2d operator
    """

    def __init__(self,
                 kernel_size,
                 stride=None,
                 padding=0,
                 odd_padding=(0, 0, 0, 0)):
        """
        Args:
            kernel_size (int or tuple): kernel size for two direction of each 
                axis. For example, (2, 3), the first 2 means will add 2 at the 
                beginning and also 2 at the end for its axis.and if a int is 
                accepted, the kernel size will be initiated as (int, int)
            stride (int or tuple): stride, the logic is the same as kernel size.
            padding (int): tuple, list or None, padding, the logic is the same 
                as kernel size. However, if you set pad_mode as "SAME_UPPER" or 
                "SAME_LOWER" mode, you can set padding as None, and the padding 
                will be computed automatically.
            odd_padding (tuple of four int): the odd paddding is the value 
                that cannot be handled by the tuple padding (w, h) mode so 
                it needs to firstly handle the input, then use the normal 
                padding method.
        """
        super(MaxPool2d, self).__init__(kernel_size, stride, padding, True,
                                        odd_padding)


class AvgPool2d(Pooling2d):

    def __init__(self,
                 kernel_size,
                 stride=None,
                 padding=0,
                 odd_padding=(0, 0, 0, 0)):
        """
        Args:
            kernel_size (int or tuple): kernel size for two direction of each 
                axis. For example, (2, 3), the first 2 means will add 2 at the 
                beginning and also 2 at the end for its axis.and if a int is 
                accepted, the kernel size will be initiated as (int, int)
            stride (int or tuple): stride, the logic is the same as kernel size.
            padding (int): tuple, list or None, padding, the logic is the same 
                as kernel size. However, if you set pad_mode as "SAME_UPPER" or 
                "SAME_LOWER" mode, you can set padding as None, and the padding 
                will be computed automatically.
            odd_padding (tuple of four int): the odd paddding is the value 
                that cannot be handled by the tuple padding (w, h) mode so 
                it needs to firstly handle the input, then use the normal 
                padding method.
        """
        super(AvgPool2d, self).__init__(kernel_size, stride, padding, False,
                                        odd_padding)


class MaxPool1d(Pooling2d):
    """
    Generate a Max Pooling 1d operator
    """

    def __init__(self,
                 kernel_size,
                 stride=None,
                 padding=0,
                 odd_padding=(0, 0, 0, 0)):
        """
        Args:
            kernel_size (int or tuple): kernel size for two direction of each 
                axis. For example, (2, 3), the first 2 means will add 2 at the 
                beginning and also 2 at the end for its axis.and if a int is 
                accepted, the kernel size will be initiated as (int, int)
            stride (int or tuple): stride, the logic is the same as kernel size.
            padding (int): tuple, list or None, padding, the logic is the same 
                as kernel size. However, if you set pad_mode as "SAME_UPPER" or 
                "SAME_LOWER" mode, you can set padding as None, and the padding 
                will be computed automatically.
            odd_padding (tuple of four int): the odd paddding is the value 
                that cannot be handled by the tuple padding (w, h) mode so 
                it needs to firstly handle the input, then use the normal 
                padding method.
        """
        if stride is None:
            stride = kernel_size
        super(MaxPool1d, self).__init__((1, kernel_size), (1, stride),
                                        (0, padding), True, odd_padding)


class AvgPool1d(Pooling2d):
    """
    Generate a Avg Pooling 1d operator
    """

    def __init__(self,
                 kernel_size,
                 stride=None,
                 padding=0,
                 odd_padding=(0, 0, 0, 0)):
        """
        Args:
            kernel_size (int or tuple): kernel size for two direction of each 
                axis. For example, (2, 3), the first 2 means will add 2 at the 
                beginning and also 2 at the end for its axis.and if a int is 
                accepted, the kernel size will be initiated as (int, int)
            stride (int or tuple): stride, the logic is the same as kernel size.
            padding (int): tuple, list or None, padding, the logic is the same 
                as kernel size. However, if you set pad_mode as "SAME_UPPER" or 
                "SAME_LOWER" mode, you can set padding as None, and the padding 
                will be computed automatically.
            odd_padding (tuple of four int): the odd paddding is the value 
                that cannot be handled by the tuple padding (w, h) mode so 
                it needs to firstly handle the input, then use the normal 
                padding method.
        """
        if stride is None:
            stride = kernel_size
        super(AvgPool1d, self).__init__((1, kernel_size), (1, stride),
                                        (0, padding), False, odd_padding)


class Tanh(Operation):
    """
    Calculates the hyperbolic tangent of the given input tensor element-wise.
    """

    def __init__(self):
        super(Tanh, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        out = singa.Tanh(x)
        if training:
            self.cache = (out,)
        return out

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.__mul__(self.cache[0], self.cache[0])
        dx = singa.MultFloat(dx, -1.0)
        dx = singa.AddFloat(dx, 1.0)
        dx *= dy
        return dx


def tanh(x):
    """
    Calculates the hyperbolic tangent of the given input tensor element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Tanh()(x)[0]


class Cos(Operation):
    """
    Calculates the cosine of the given input tensor, element-wise.
    """

    def __init__(self):
        super(Cos, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Cos(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Sin(self.input)
        dx = singa.MultFloat(dx, -1.0)
        dx *= dy
        return dx


def cos(x):
    """
    Calculates the cosine of the given input tensor, element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """

    return Cos()(x)[0]


class Cosh(Operation):
    """
    Calculates the hyperbolic cosine of the given input tensor element-wise.
    """

    def __init__(self):
        super(Cosh, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Cosh(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Sinh(self.input)
        dx *= dy
        return dx


def cosh(x):
    """
    Calculates the hyperbolic cosine of the given input tensor element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Cosh()(x)[0]


class Acos(Operation):
    """
    Calculates the arccosine (inverse of cosine) of the given input tensor, 
    element-wise.
    """

    def __init__(self):
        super(Acos, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Acos(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Square(self.input)
        dx = singa.MultFloat(dx, -1.0)
        dx = singa.AddFloat(dx, 1.0)
        dx = singa.PowFloat(dx, -0.5)
        dx = singa.MultFloat(dx, -1.0)
        dx *= dy
        return dx


def acos(x):
    """
    Calculates the arccosine (inverse of cosine) of the given input tensor, 
    element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Acos()(x)[0]


class Acosh(Operation):
    """
    Calculates the hyperbolic arccosine of the given input tensor element-wise.
    """

    def __init__(self):
        super(Acosh, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Acosh(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.SubFloat(self.input, 1.0)
        dx = singa.Sqrt(dx)
        temp = singa.AddFloat(self.input, 1.0)
        temp = singa.Sqrt(temp)
        dx = singa.__mul__(dx, temp)
        dx = singa.PowFloat(dx, -1.0)
        dx *= dy
        return dx


def acosh(x):
    """
    Calculates the hyperbolic arccosine of the given input tensor element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Acosh()(x)[0]


class Sin(Operation):
    """
    Calculates the sine of the given input tensor, element-wise.
    """

    def __init__(self):
        super(Sin, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Sin(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Cos(self.input)
        dx *= dy
        return dx


def sin(x):
    """
    Calculates the sine of the given input tensor, element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Sin()(x)[0]


class Sinh(Operation):
    """
    Calculates the hyperbolic sine of the given input tensor element-wise.
    """

    def __init__(self):
        super(Sinh, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Sinh(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Cosh(self.input)
        dx *= dy
        return dx


def sinh(x):
    """
    Calculates the hyperbolic sine of the given input tensor element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Sinh()(x)[0]


class Asin(Operation):
    """
    Calculates the arcsine (inverse of sine) of the given input tensor, element-wise.
    """

    def __init__(self):
        super(Asin, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Asin(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Square(self.input)
        dx = singa.MultFloat(dx, -1.0)
        dx = singa.AddFloat(dx, 1.0)
        dx = singa.PowFloat(dx, -0.5)
        dx *= dy
        return dx


def asin(x):
    """
    Calculates the arcsine (inverse of sine) of the given input tensor, element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """

    return Asin()(x)[0]


class Asinh(Operation):
    """
    Calculates the hyperbolic arcsine of the given input tensor element-wise.
    """

    def __init__(self):
        super(Asinh, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Asinh(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Square(self.input)
        dx = singa.AddFloat(dx, 1.0)
        dx = singa.PowFloat(dx, -0.5)
        dx *= dy
        return dx


def asinh(x):
    """
    Calculates the hyperbolic arcsine of the given input tensor element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Asinh()(x)[0]


class Tan(Operation):
    """
    Insert single-dimensional entries to the shape of an input tensor (data). 
    """

    def __init__(self):
        super(Tan, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Tan(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Cos(self.input)
        dx = singa.Square(dx)
        dx = singa.PowFloat(dx, -1.0)
        dx *= dy
        return dx


def tan(x):
    """
    Calculates the tangent of the given input tensor, element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Tan()(x)[0]


class Atan(Operation):
    """
    Calculates the arctangent (inverse of tangent) of the given input tensor, element-wise.
    """

    def __init__(self):
        super(Atan, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Atan(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Square(self.input)
        dx = singa.AddFloat(dx, 1.0)
        dx = singa.PowFloat(dx, -1.0)
        dx *= dy
        return dx


def atan(x):
    """
    Calculates the arctangent (inverse of tangent) of the given input tensor, element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Atan()(x)[0]


class Atanh(Operation):
    """
    Calculates the hyperbolic arctangent of the given input tensor element-wise.
    """

    def __init__(self):
        super(Atanh, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        return singa.Atanh(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Square(self.input)
        dx = singa.MultFloat(dx, -1.0)
        dx = singa.AddFloat(dx, 1.0)
        dx = singa.PowFloat(dx, -1.0)
        dx *= dy
        return dx


def atanh(x):
    """
    Calculates the hyperbolic arctangent of the given input tensor element-wise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Atanh()(x)[0]


class Sigmoid(Operation):
    """
    `y = 1 / (1 + exp(-x))`, is applied to the tensor elementwise.
    """

    def __init__(self):
        super(Sigmoid, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        out = singa.Sigmoid(x)
        if training:
            self.cache = (out,)
        return out

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.MultFloat(self.cache[0], -1.0)
        dx = singa.AddFloat(dx, 1.0)
        dx = singa.__mul__(self.cache[0], dx)
        dx *= dy
        return dx


def sigmoid(x):
    """
    `y = 1 / (1 + exp(-x))`, is applied to the tensor elementwise.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Sigmoid()(x)[0]


class Mul(Operation):
    """
    Performs element-wise binary multiplication (with Numpy-style broadcasting 
    support).        
    """

    def __init__(self):
        super(Mul, self).__init__()

    def forward(self, a, b):
        """
        Return `np.multiply(a,b)`, where a and b are CTensor.
        """
        # todo we cannot support mul op for int tensors
        _a, _b = a, b
        dtype0 = _a.data_type()
        dtype1 = _b.data_type()
        if dtype0 == singa.kInt or dtype1 == singa.kInt:
            _a = a.AsType(singa.kFloat32)
            _b = b.AsType(singa.kFloat32)
            res = singa.__mul__(_a, _b)
            res = res.AsType(singa.kInt)
        else:
            res = singa.__mul__(_a, _b)
        if training:
            self.input = (_a, _b)
            self.shape0 = list(_a.shape())
            self.shape1 = list(_b.shape())
            self.shape3 = list(res.shape())
        return res

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            a tuple for (da, db), da is data for dL / da, db is data
                for dL / db.
        """
        dx0 = singa.__mul__(dy, self.input[1])
        dx1 = singa.__mul__(dy, self.input[0])
        if (type(dy) == float) or self.shape0 == self.shape1:
            assert self.shape0 == self.shape1, ('should have same shape')
            return dx0, dx1
        # handle broadcast
        dx0 = back_broadcast(self.shape3, self.shape0, dx0)
        dx1 = back_broadcast(self.shape3, self.shape1, dx1)
        return dx0, dx1


def mul(x, y):
    """
    Return `np.multiply(x,y)`, where a and b are Tensor.
    """
    return Mul()(x, y)[0]


class Unsqueeze(Operation):
    """
    Insert single-dimensional entries to the shape of an input tensor (data). 
    """

    def __init__(self, axis):
        """
        Args:
            axis (list of int): the dimensions to be inserted.
        """
        super(Unsqueeze, self).__init__()
        if (type(axis) is int):
            self.axis = list(axis)
        else:
            self.axis = axis

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        self.cache = x.shape()
        cur = list(self.cache)
        # todo, need optimize after we have scalar tensor
        if len(self.cache) == 1 and self.axis == [0]:
            return x
        for i in self.axis:
            cur.insert(i, 1)
        return singa.Reshape(x, cur)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        return singa.Reshape(dy, self.cache)


def unsqueeze(x, axis=-1):
    """
    Insert single-dimensional entries to the shape of an input tensor (data). 
    Args:
        x (Tensor): Input tensor
        axis (list of int): the dimensions to be inserted.
    Returns: 
        Tensor, the output
    """
    return Unsqueeze(axis)(x)[0]


class Transpose(Operation):
    """
    Transpose the input tensor similar to numpy.transpose. 
    """

    def __init__(self, perm):
        """
        Args:
            perm (list of ints): A list of integers. By default, reverse the 
                dimensions, otherwise permute the axes according to the values given.
        """
        super(Transpose, self).__init__()
        self.perm = list(perm)

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        return singa.Transpose(x, self.perm)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        cur = []
        for i in range(len(self.perm)):
            cur += [self.perm.index(i)]
        return singa.Transpose(dy, cur)


def transpose(x, shape):
    """
    Transpose the input tensor similar to numpy.transpose. 
    Args:
        x (Tensor): Input tensor
        perm (list of ints): A list of integers. By default, reverse the 
            dimensions, otherwise permute the axes according to the values given.
    Returns: 
        Tensor, the output
    """
    return Transpose(shape)(x)[0]


def add_all(*xs):
    assert len(xs) > 2
    y = add(xs[0], xs[1])
    for x in xs[2:]:
        y = add(y, x)
    return


class RNN_Base(Layer):

    def __init__(self):
        raise NotImplementedError

    def __call__(self):
        raise NotImplementedError

    def step_forward(self,
                     x=None,
                     h=None,
                     c=None,
                     Wx=None,
                     Wh=None,
                     Bx=None,
                     Bh=None,
                     b=None):
        raise NotImplementedError


class RNN(RNN_Base):
    """
    Generate a RNN operator
    """

    def __init__(
            self,
            input_size,
            hidden_size,
            num_layers=1,
            nonlinearity="tanh",
            bias=True,
            batch_first=False,
            dropout=0,
            bidirectional=False,
    ):
        """
        Args:
            input_size (int):  The number of expected features in the input x
            hidden_size (int): The number of features in the hidden state h
            num_layers (int):  Number of recurrent layers. Default: 1
            nonlinearity (string): The non-linearity to use. Default: 'tanh'
            bias (bool):  If False, then the layer does not use bias weights. 
                Default: True
            batch_first (bool):  If True, then the input and output tensors 
                are provided as (batch, seq, feature). Default: False
            dropout (float): If non-zero, introduces a Dropout layer on the 
                outputs of each RNN layer except the last layer, with dropout 
                probability equal to dropout. Default: 0
            bidirectional (bool): If True, becomes a bidirectional RNN. 
                Default: False
        """
        self.nonlinearity = nonlinearity

        Wx_shape = (input_size, hidden_size)
        self.Wx = Tensor(shape=Wx_shape, requires_grad=True, stores_grad=True)
        self.Wx.gaussian(0.0, 1.0)

        Wh_shape = (hidden_size, hidden_size)
        self.Wh = Tensor(shape=Wh_shape, requires_grad=True, stores_grad=True)
        self.Wh.gaussian(0.0, 1.0)

        B_shape = (hidden_size,)
        self.b = Tensor(shape=B_shape, requires_grad=True, stores_grad=True)
        self.b.set_value(0.0)

        self.params = (self.Wx, self.Wh, self.b)

    def __call__(self, xs, h0):
        # xs: a tuple or list of input tensors
        if not isinstance(xs, tuple):
            xs = tuple(xs)
        inputs = xs + (h0,)
        self.device_check(*inputs)
        # self.device_check(inputs[0], *self.params)
        self.device_check(inputs[0], self.Wx, self.Wh, self.b)
        batchsize = xs[0].shape[0]
        out = []
        h = self.step_forward(xs[0], h0, self.Wx, self.Wh, self.b)
        out.append(h)
        for x in xs[1:]:
            assert x.shape[0] == batchsize
            h = self.step_forward(x, h, self.Wx, self.Wh, self.b)
            out.append(h)
        return out, h

    def step_forward(self, x, h, Wx, Wh, b):
        y2 = matmul(h, Wh)
        y1 = matmul(x, Wx)
        y = add(y2, y1)
        y = add_bias(y, b, axis=0)
        if self.nonlinearity == "tanh":
            y = tanh(y)
        elif self.nonlinearity == "relu":
            y = relu(y)
        else:
            raise ValueError
        return y


class LSTM(RNN_Base):
    """
    Generate a LSTM operator
    """

    def __init__(
            self,
            input_size,
            hidden_size,
            nonlinearity="tanh",
            num_layers=1,
            bias=True,
            batch_first=False,
            dropout=0,
            bidirectional=False,
    ):
        """
        Args:
            input_size (int):  The number of expected features in the input x
            hidden_size (int): The number of features in the hidden state h
            num_layers (int):  Number of recurrent layers. Default: 1
            nonlinearity (string): The non-linearity to use. Default: 'tanh'
            bias (bool):  If False, then the layer does not use bias weights. 
                Default: True
            batch_first (bool):  If True, then the input and output tensors 
                are provided as (batch, seq, feature). Default: False
            dropout (float): If non-zero, introduces a Dropout layer on the 
                outputs of each RNN layer except the last layer, with dropout 
                probability equal to dropout. Default: 0
            bidirectional (bool): If True, becomes a bidirectional RNN. 
                Default: False
        """
        self.nonlinearity = nonlinearity

        Wx_shape = (input_size, hidden_size)
        self.Wx = []
        for i in range(4):
            w = Tensor(shape=Wx_shape, requires_grad=True, stores_grad=True)
            w.gaussian(0.0, 0.01)
            self.Wx.append(w)

        Wh_shape = (hidden_size, hidden_size)
        self.Wh = []
        for i in range(4):
            w = Tensor(shape=Wh_shape, requires_grad=True, stores_grad=True)
            w.gaussian(0.0, 0.01)
            self.Wh.append(w)

        Bx_shape = (hidden_size,)
        self.Bx = []
        for i in range(4):
            b = Tensor(shape=Bx_shape, requires_grad=True, stores_grad=True)
            b.set_value(0.0)
            self.Bx.append(b)

        self.Bh = []
        for i in range(4):
            b = Tensor(shape=Bx_shape, requires_grad=True, stores_grad=True)
            b.set_value(0.0)
            self.Bh.append(b)

        self.params = self.Wx + self.Wh + self.Bx + self.Bh

    def __call__(self, xs, h0_c0):
        # xs: a tuple or list of input tensors
        # h0_c0: a tuple of (h0, c0)
        h0, c0 = h0_c0
        if not isinstance(xs, list):
            xs = list(xs)
        inputs = xs + list((h0, c0))
        self.device_check(*inputs)
        # self.device_check(inputs[0], *self.params)
        self.device_check(inputs[0], *(self.Wx + self.Wh + self.Bx + self.Bh))
        batchsize = xs[0].shape[0]
        out = []
        h, c = self.step_forward(xs[0], h0, c0, self.Wx, self.Wh, self.Bx,
                                 self.Bh)
        out.append(h)
        for x in xs[1:]:
            assert x.shape[0] == batchsize
            h, c = self.step_forward(x, h, c, self.Wx, self.Wh, self.Bx,
                                     self.Bh)
            out.append(h)
        return out, h, c

    def step_forward(self, x, h, c, Wx, Wh, Bx, Bh):
        y1 = matmul(x, Wx[0])
        y1 = add_bias(y1, Bx[0], axis=0)
        y2 = matmul(h, Wh[0])
        y2 = add_bias(y2, Bh[0], axis=0)
        i = add(y1, y2)
        i = sigmoid(i)

        y1 = matmul(x, Wx[1])
        y1 = add_bias(y1, Bx[1], axis=0)
        y2 = matmul(h, Wh[1])
        y2 = add_bias(y2, Bh[1], axis=0)
        f = add(y1, y2)
        f = sigmoid(f)

        y1 = matmul(x, Wx[2])
        y1 = add_bias(y1, Bx[2], axis=0)
        y2 = matmul(h, Wh[2])
        y2 = add_bias(y2, Bh[2], axis=0)
        o = add(y1, y2)
        o = sigmoid(o)

        y1 = matmul(x, Wx[3])
        y1 = add_bias(y1, Bx[3], axis=0)
        y2 = matmul(h, Wh[3])
        y2 = add_bias(y2, Bh[3], axis=0)
        g = add(y1, y2)
        g = tanh(g)

        cout1 = mul(f, c)
        cout2 = mul(i, g)
        cout = add(cout1, cout2)

        hout = tanh(cout)
        hout = mul(o, hout)
        return hout, cout


class Abs(Operation):
    """
    `y = abs(x)`, is applied to the tensor elementwise.
    """

    def forward(self, a):
        """
        Return `abs(a)`, where a is CTensor.
        """
        if training:
            self.input = a
        return singa.Abs(a)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Sign(self.input)
        dx *= dy
        return dx


def abs(a):
    """
    Return abs(a), where a is Tensor.
    """
    return Abs()(a)[0]


class Exp(Operation):
    """
    `y = exp(x)`, is applied to the tensor elementwise.
    """

    def forward(self, a):
        """
        Return `exp(a)`, where a is Tensor.
        """
        if training:
            self.input = a
        return singa.Exp(a)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Exp(self.input)
        dx *= dy
        return dx


def exp(a):
    """
    Return `exp(a)`, where a is Tensor.
    """
    return Exp()(a)[0]


class LeakyRelu(Operation):
    """
    `f(x) = alpha * x` for x < 0, `f(x) = x` for x >= 0, is applied to the tensor elementwise.
    """

    def __init__(self, a):
        """
        Args:
            a (float): Coefficient of leakage.
        """
        super(LeakyRelu, self).__init__()
        self.a = a

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = x
        x1 = singa.LTFloat(x, 0.0)
        x1 = singa.__mul__(x, x1)
        x1 = singa.MultFloat(x1, self.a)
        x2 = singa.ReLU(x)
        x1 = singa.__add__(x1, x2)
        return x1

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        # TODO(wangwei) check the correctness
        dx1 = singa.GTFloat(self.input, 0.0)
        dx2 = singa.LTFloat(self.input, 0.0)
        dx2 = singa.MultFloat(dx2, self.a)
        dx = singa.__add__(dx1, dx2)
        dx *= dy
        return dx


def leakyrelu(x, a=0.01):
    """
    `f(x) = alpha * x` for x < 0, `f(x) = x` for x >= 0 is applied to the tensor 
    elementwise.
    Args:
        x (Tensor): Input tensor
        a (float): Coefficient of leakage, default to 0.01.
    Returns: 
        Tensor, the output
    """
    return LeakyRelu(a)(x)[0]


class Sign(Operation):
    """
    Calculate the sign of the given input tensor element-wise. If input > 0, 
    output 1. if input < 0, output -1. if input == 0, output 0.
    """

    def __init__(self):
        super(Sign, self).__init__()

    def forward(self, a):
        """
        Args:
            a (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.input = a
        return singa.Sign(a)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.MultFloat(dy, 0.0)
        return dx


def sign(a):
    """
    Calculate the sign of the given input tensor element-wise. If input > 0, 
    output 1. if input < 0, output -1. if input == 0, output 0.
    Args:
        a (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Sign()(a)[0]


class Pow(Operation):
    """
    `f(x) = a^b`, is applied to the tensor elementwise.
    """

    def __init__(self):
        super(Pow, self).__init__()

    def forward(self, a, b):
        """
        Return `a^b`, where a and b are CTensor.
        """
        res = singa.Pow(a, b)
        if training:
            self.input = (a, b)
            self.shape0 = list(a.shape())
            self.shape1 = list(b.shape())
            self.shape3 = list(res.shape())
        return res

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            a tuple for (da, db), da is data for dL / da, db is data
                for dL / db.
        """
        da1 = singa.__mul__(
            self.input[1],
            singa.Pow(self.input[0], singa.SubFloat(self.input[1], 1.0)))
        dx0 = singa.__mul__(da1, dy)
        db1 = singa.__mul__(singa.Pow(self.input[0], self.input[1]),
                            singa.Log(self.input[0]))
        dx1 = singa.__mul__(db1, dy)
        if (type(dy) == float) or self.shape0 == self.shape1:
            assert self.shape0 == self.shape1, ('should have same shape')
            return dx0, dx1
        # handle broadcast
        dx0 = back_broadcast(self.shape3, self.shape0, dx0)
        dx1 = back_broadcast(self.shape3, self.shape1, dx1)
        return dx0, dx1


def pow(a, b):
    """
    Return `a^b`, where a and b are Tensor.
    """
    return Pow()(a, b)[0]


class SoftSign(Operation):
    """
    Calculates the softsign `(x/(1+|x|))` of the given input tensor element-wise.
    """

    def __init__(self):
        super(SoftSign, self).__init__()

    def forward(self, x):
        """
        Return `(x/(1+|x|))`, where x is CTensor.
        """
        # y = x / (1 + np.abs(x))
        if training:
            self.input = x
        x1 = singa.AddFloat(singa.Abs(x), 1.0)
        y = singa.__div__(x, x1)

        return y

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.AddFloat(singa.Abs(self.input), 1.0)
        dx = singa.PowFloat(singa.Square(dx), -1.0)
        dx = singa.__mul__(dy, dx)
        return dx


def softsign(x):
    """
    Return `(x/(1+|x|))`, where x is Tensor.
    """
    return SoftSign()(x)[0]


class Sqrt(Operation):
    """
    `y = x^0.5`, is applied to the tensor elementwise.
    """

    def __init__(self):
        super(Sqrt, self).__init__()

    def forward(self, x):
        """
        Return `x^0.5`, where x is CTensor.
        """
        if training:
            self.input = x
        return singa.Sqrt(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.PowFloat(self.input, -0.5)
        dx = singa.MultFloat(dx, 0.5)
        dx = singa.__mul__(dy, dx)
        return dx


def sqrt(x):
    """
    Return `x^0.5`, where x is Tensor.
    """
    return Sqrt()(x)[0]


class SoftPlus(Operation):
    """
    `y = ln(exp(x) + 1)` is applied to the tensor elementwise.
    """

    def __init__(self):
        super(SoftPlus, self).__init__()

    def forward(self, x):
        """
        Return `ln(exp(x) + 1)`, where x is CTensor.
        """
        #f(x) = ln(exp(x) + 1)
        if training:
            self.input = x
        x1 = singa.AddFloat(singa.Exp(x), 1.0)
        y = singa.Log(x1)
        return y

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.Exp(singa.MultFloat(self.input, -1.0))
        dx = singa.PowFloat(singa.AddFloat(dx, 1.0), -1.0)
        dx = singa.__mul__(dy, dx)
        return dx


def softplus(x):
    """
    Return `ln(exp(x) + 1)`, where x is Tensor.
    """
    return SoftPlus()(x)[0]


class Sub(Operation):
    """
    Performs element-wise binary subtraction (with Numpy-style broadcasting 
    support).
    """

    def __init__(self):
        super(Sub, self).__init__()

    def forward(self, a, b):
        """
        Return `a-b`, where x is CTensor.
        """
        res = singa.__sub__(a, b)
        if training:
            self.shape0 = list(a.shape())
            self.shape1 = list(b.shape())
            self.shape3 = list(res.shape())
        return res

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            a tuple for (da, db), da is data for dL / da, db is data
                for dL / db.
        """
        dx0 = dy
        dx1 = singa.MultFloat(dy, -1.0)
        if (type(dy) == float) or self.shape0 == self.shape1:
            assert self.shape0 == self.shape1, ('should have same shape')
            return dx0, dx1
        # handle broadcast
        dx0 = back_broadcast(self.shape3, self.shape0, dx0)
        dx1 = back_broadcast(self.shape3, self.shape1, dx1)
        return dx0, dx1


def sub(a, b):
    """
    Return a-b, where a and b are Tensor.
    """
    return Sub()(a, b)[0]


# optimize min to support multi inputs
class Min(Operation):
    """
    Element-wise min of each of the input tensors (with Numpy-style 
    broadcasting support).
    """

    def __init__(self):
        super(Min, self).__init__()
        self.masks = []

    def _min(self, a, b):
        """
        Args:
            a (CTensor): First operand
            b (CTensor): Second operand
        Returns: 
            CTensor, the output
            tuple of CTensor, mask tensor
        """
        m = singa.__sub__(a, b)
        mask0 = singa.LEFloat(m, 0)
        mask1 = singa.GTFloat(m, 0)
        res = singa.__add__(singa.__mul__(mask0, a), singa.__mul__(mask1, b))
        return res, (mask0, mask1)

    def forward(self, *x):
        """
        Args:
            *x (a list of CTensor): List of tensors for max.
        Returns: 
            CTensor, the output
        """
        assert (len(x) > 0)
        self.l = len(x)
        if len(x) == 1:
            res, masks = self._min(x[0], x[0])
            self.masks.append(masks)
            return x[0]
        res, masks = self._min(x[0], x[1])
        self.masks.append(masks)
        for i in range(2, len(x)):
            res, masks = self._min(res, x[i])
            self.masks.append(masks)
        return res

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            a tuple for (*dx), dx is data for dL / dx.
        """
        if self.l == 1:
            return self.masks[0][0]
        else:
            ret = []
            cumulation = None
            for mask0, mask1 in self.masks[::-1]:
                if not cumulation:
                    ret.insert(0, mask1)
                    cumulation = mask0
                else:
                    ret.insert(0, singa.__mul__(cumulation, mask1))
                    cumulation = singa.__mul__(cumulation, mask0)
            ret.insert(0, cumulation)
            return tuple(ret)


def min(*l):
    """
    Element-wise min of each of the input tensors (with Numpy-style 
    broadcasting support).
    Args:
        *x (a list of Tensor): List of tensors for max.
    Returns: 
        Tensor, the output
    """
    return Min()(*l)[0]


class Log(Operation):
    """
    `y = log(x)`, is applied to the tensor elementwise.
    """

    def __init__(self):
        super(Log, self).__init__()

    def forward(self, x):
        """
        Return `log(x)`, where x is CTensor.
        """
        if training:
            self.input = x
        return singa.Log(x)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        dx = singa.PowFloat(self.input, -1)
        dx = singa.__mul__(dy, dx)
        return dx


def log(x):
    """
    Return log(x), where x is Tensor.
    """
    return Log()(x)[0]


class HardSigmoid(Operation):
    """
    `y = max(0, min(1, alpha * x + beta))`, is applied to the tensor elementwise.
    """

    def __init__(self, alpha=0.2, gamma=0.5):
        """
        Args:
            alpha (float): Value of alpha.
            gamma (float): Value of beta.
        """
        super(HardSigmoid, self).__init__()
        self.alpha = alpha
        self.gamma = gamma

    def forward(self, x):
        """
        Args:
            x (CTensor): matrix
        Returns:
            a CTensor for the result
        """
        x = singa.AddFloat(singa.MultFloat(x, self.alpha), self.gamma)
        if training:
            self.cache = x

        x = singa.ReLU(x)
        mask1 = singa.LTFloat(x, 1.0)
        mask2 = singa.GEFloat(x, 1.0)

        ans = singa.__add__(singa.__mul__(x, mask1), mask2)
        return singa.ReLU(ans)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        mask0 = singa.GTFloat(self.cache, 0.0)
        mask1 = singa.LTFloat(self.cache, 1.0)
        mask = singa.__mul__(mask0, mask1)
        return singa.__mul__(singa.MultFloat(mask, self.alpha), dy)


def hardsigmoid(x, alpha=0.2, gamma=0.5):
    """
    `y = max(0, min(1, alpha * x + beta))`, is applied to the tensor elementwise.
    Args:
        x (Tensor): matrix
        alpha (float): Value of alpha.
        gamma (float): Value of beta.        
    Returns:
        a Tensor for the result
    """
    return HardSigmoid(alpha, gamma)(x)[0]


class Squeeze(Operation):
    """
    Remove single-dimensional entries from the shape of a tensor. Takes a 
    parameter axes with a list of axes to squeeze. If axes is not provided, 
    all the single dimensions will be removed from the shape. If an axis is 
    selected with shape entry not equal to one, an error is raised.
    """

    def __init__(self, axis=[]):
        """
        Args:
            axis (list of ints): List of integers indicating the dimensions 
                to squeeze. Negative value means counting dimensions from 
                the back. Accepted range is [-r, r-1] where r = rank(data).
        """
        super(Squeeze, self).__init__()
        self.axis = axis

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        self.cache = x.shape()
        newshape = []
        if (self.axis == []):
            newshape = list(filter(lambda i: i != 1, self.cache))
        else:
            for id, i in enumerate(self.axis):
                assert i < len(self.cache)
                self.axis[id] = i % len(self.cache)
                assert self.cache[
                    i] == 1, "the length of axis {} is {}, which should be 1".format(
                        i, self.cache[i])
            for ind, v in enumerate(self.cache):
                if ind not in self.axis:
                    newshape.append(v)
        # todo, need optimize after we have scalar tensor
        if newshape == []:
            return x
        return singa.Reshape(x, newshape)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        return singa.Reshape(dy, self.cache)


def squeeze(x, axis=[]):
    """
    Remove single-dimensional entries from the shape of a tensor. Takes a 
    parameter axes with a list of axes to squeeze. If axes is not provided, 
    all the single dimensions will be removed from the shape. If an axis is 
    selected with shape entry not equal to one, an error is raised.
    Args:
        x (Tensor): Input tensor
        axis (list of ints): List of integers indicating the dimensions 
            to squeeze. Negative value means counting dimensions from 
            the back. Accepted range is [-r, r-1] where r = rank(data).
    Returns: 
        Tensor, the output
    """
    return Squeeze(axis)(x)[0]


class Div(Operation):
    """
    Performs element-wise binary division (with Numpy-style broadcasting support).
    """

    def __init__(self):
        super(Div, self).__init__()

    def forward(self, a, b):
        """
        Return `np.div(a,b)`, where a and b are CTensor.
        """
        res = singa.__mul__(a, singa.PowFloat(b, -1.0))
        # res = singa.__div__(a, b)
        if training:
            self.input = (singa.MultFloat(a, -1.0), singa.PowFloat(b, -1.0)
                         )  # -a, 1/b
            self.shape0 = list(a.shape())
            self.shape1 = list(b.shape())
            self.shape3 = list(res.shape())
        return res

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            a CTensor tuple for (da, db), da is data for dL / da, db is data
                for dL / db.
        """
        #dy/dx_0 = b^(-1)
        #dy/dx_1 = (-a)*b^(-2)
        dx0 = singa.__mul__(dy, self.input[1])
        dx1 = singa.__mul__(self.input[0], singa.PowFloat(self.input[1], 2.0))
        dx1 = singa.__mul__(dy, dx1)
        if (type(dy) == float) or self.shape0 == self.shape1:
            assert self.shape0 == self.shape1, ('should have same shape')
            return dx0, dx1
        # handle broadcast
        dx0 = back_broadcast(self.shape3, self.shape0, dx0)
        dx1 = back_broadcast(self.shape3, self.shape1, dx1)
        return dx0, dx1


def div(a, b):
    """
    Return `np.div(a,b)`, where a and b are Tensor.
    """
    return Div()(a, b)[0]


class Shape(Operation):
    """
    Takes a tensor as input and outputs a tensor containing the shape of the 
    input tensor.
    """

    def __init__(self):
        super(Shape, self).__init__()

    def forward(self, x):
        """
        Args:
            x (CTensor): Input tensor
        Returns: 
            CTensor, the output
        """
        cur = list(x.shape())
        cur = tensor.from_numpy(np.array(cur))
        cur.to_device(x.device())
        return cur.data

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            list of int, the shape of dy
        """
        return list(dy.shape())


def shape(x):
    """
    Takes a tensor as input and outputs a tensor containing the shape of the 
    input tensor.
    Args:
        x (Tensor): Input tensor
    Returns: 
        Tensor, the output
    """
    return Shape()(x)[0]


# optimize max to support multi inputs
class Max(Operation):
    """
    Element-wise max of each of the input tensors (with Numpy-style 
    broadcasting support). 
    """

    def __init__(self):
        super(Max, self).__init__()
        self.masks = []

    def _max(self, a, b):
        """
        Args:
            a (CTensor): First operand
            b (CTensor): Second operand
        Returns: 
            CTensor, the output
            tuple of CTensor, mask tensor
        """
        m = singa.__sub__(a, b)
        mask0 = singa.GEFloat(m, 0)
        mask1 = singa.LTFloat(m, 0)
        res = singa.__add__(singa.__mul__(mask0, a), singa.__mul__(mask1, b))
        return res, (mask0, mask1)

    def forward(self, *x):
        """
        Args:
            *x (a list of CTensor): List of tensors for max.
        Returns: 
            CTensor, the output
        """
        assert (len(x) > 0)
        self.l = len(x)
        if len(x) == 1:
            res, masks = self._max(x[0], x[0])
            self.masks.append(masks)
            return x[0]
        res, masks = self._max(x[0], x[1])
        self.masks.append(masks)
        for i in range(2, len(x)):
            res, masks = self._max(res, x[i])
            self.masks.append(masks)
        return res

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            a tuple for (*dx), dx is data for dL / dx.
        """
        if self.l == 1:
            return self.masks[0][0]
        else:
            ret = []
            cumulation = None
            for mask0, mask1 in self.masks[::-1]:
                if not cumulation:
                    ret.insert(0, mask1)
                    cumulation = mask0
                else:
                    ret.insert(0, singa.__mul__(cumulation, mask1))
                    cumulation = singa.__mul__(cumulation, mask0)
            ret.insert(0, cumulation)
            return tuple(ret)


def max(*l):
    """
    Element-wise max of each of the input tensors (with Numpy-style broadcasting support). 
    Args:
        *x (a list of Tensor): List of tensors for max.
    Returns: 
        Tensor, the output
    """
    return Max()(*l)[0]


class And(Operation):
    """
    Returns the tensor resulted from performing the and logical operation elementwise on the input tensors A and B (with Numpy-style broadcasting support).
    """

    def __init__(self):
        super(And, self).__init__()

    def forward(self, a, b):
        """
        Return `np.logical_and(a,b)`, where a and b are CTensor.
        """
        m = singa.__mul__(a, b)
        cur = singa.PowFloat(singa.Sign(m), 2)

        return cur

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no gradient for backward function')


def _and(a, b):
    """
    Return `np.logical_and(a,b)`, where a and b are Tensor.
    """
    return And()(a, b)[0]


class Or(Operation):
    """
    Returns the tensor resulted from performing the or logical operation elementwise on the input tensors A and B (with Numpy-style broadcasting support).
    """

    def __init__(self):
        super(Or, self).__init__()

    def forward(self, a, b):
        """
        Return `np.logical_or(a,b)`, where a and b are CTensor.
        """
        m = singa.__add__(singa.PowFloat(singa.Sign(a), 2.0),
                          singa.PowFloat(singa.Sign(b), 2.0))
        cur = singa.Sign(m)

        return cur

    def backward(self, dy):
        """
        Args:
            dy (CTensor): data for the `dL / dy`, L is the loss.
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no gradient for backward function')


def _or(a, b):
    """
    Return np.logical_or(a,b), where a and b are Tensor.
    """
    return Or()(a, b)[0]


class Not(Operation):
    """
    Returns the negation of the input tensor element-wise.
    """

    def __init__(self):
        super(Not, self).__init__()

    def forward(self, x):
        """
        Return `np.logical_not(x)`, where x is CTensor.
        """
        mask0 = singa.GEFloat(x, 0)
        mask1 = singa.LEFloat(x, 0)
        cur = singa.__mul__(mask0, mask1)

        return cur

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no gradient for backward function')


def _not(x):
    """
    Return `np.logical_not(x)`, where x is Tensor.
    """
    return Not()(x)[0]


class Xor(Operation):
    """
    Performing the xor logical operation elementwise on the input tensors A and B (with Numpy-style broadcasting support).
    """

    def __init__(self):
        super(Xor, self).__init__()

    def forward(self, a, b):
        """
        Return `np.logical_xor(a,b)`, where a and b are CTensor.
        """
        m = singa.__sub__(singa.PowFloat(singa.Sign(a), 2.0),
                          singa.PowFloat(singa.Sign(b), 2.0))
        cur = singa.PowFloat(singa.Sign(m), 2.0)

        return cur

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no gradient for backward function')


def _xor(a, b):
    """
    Return `np.logical_xor(a,b)`, where a and b are Tensor.
    """
    return Xor()(a, b)[0]


class Negative(Operation):
    """
    `y = -x`, is applied to the tensor elementwise.
    """

    def __init__(self):
        super(Negative, self).__init__()

    def forward(self, x):
        """
        Return `-x`, where x is CTensor.
        """
        #y=-x
        return singa.MultFloat(x, -1)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        return singa.MultFloat(dy, -1)


def negative(x):
    """
    Return `-x`, where x is Tensor.
    """
    return Negative()(x)[0]


class Reciprocal(Operation):
    """
    `y = 1/x`, is applied to the tensor elementwise.
    """

    def __init__(self):
        super(Reciprocal, self).__init__()

    def forward(self, x):
        """
        Return `1/x`, where x is CTensor.
        """
        #y=1/x elementwise
        if training:
            self.input = x

        return singa.PowFloat(x, -1)

    def backward(self, dy):
        """
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        #dy/dx = -1/x**2
        dx = singa.MultFloat(singa.PowFloat(self.input, -2), -1)
        return singa.__mul__(dy, dx)


def reciprocal(x):
    """
    Return 1/x, where x is Tensor.
    """
    return Reciprocal()(x)[0]


class Gemm(Operation):
    """
    Init a General Matrix multiplication(Gemm) operator. Compute `Y = alpha * 
    A' * B' + beta * C`, where input tensor A has shape (M, K) or (K, M), input 
    tensor B has shape (K, N) or (N, K), input tensor C is broadcastable to 
    shape (M, N), and output tensor Y has shape (M, N).
    `A' = transpose(A)` if transA else A
    `B' = transpose(B)` if transB else B
    """

    def __init__(self, alpha=1.0, beta=1.0, transA=0, transB=0):
        """
        Args:
            alpha (float): Scalar multiplier for the product of input tensors 
                A * B.
            beta (float): Scalar multiplier for input tensor C.
            ransA (int): Whether A should be transposed
            transB (int): Whether B should be transposed
        Returns: 
            CTensor, the output
        """
        super(Gemm, self).__init__()
        self.alpha = alpha
        self.beta = beta
        self.transA = transA
        self.transB = transB

    def forward(self, A, B, C=None):
        """
        forward propogation of Gemm
        Args:
            A (CTensor): The shape of A should be (M, K) if transA is 0, or 
                (K, M) if transA is non-zero.
            B (CTensor): The shape of B should be (K, N) if transB is 0, or 
                (N, K) if transB is non-zero.
            C (CTensor): (optional), Optional input tensor C. If not specified, 
                the computation is done as if C is a scalar 0. The shape of C 
                should be unidirectional broadcastable to (M, N).
        Returns: 
            tensor, the output
        """
        _A = singa.DefaultTranspose(A) if self.transA == 1 else A
        _B = singa.DefaultTranspose(B) if self.transB == 1 else B
        if training:
            self.inputs = (_A, _B, C)
        tmpM = singa.MultFloat(singa.Mult(_A, _B), self.alpha)
        if C:
            tmpM = singa.__add__(tmpM, singa.MultFloat(C, self.beta))
        return tmpM

    def backward(self, dy):
        """
        backward propogation of Gemm
        Args:
            dy (CTensor): The shape of A should be (M, K) if transA is 0, or (K, M) if transA is non-zero.
        Returns: 
            CTensor, the gradient over A
            CTensor, the gradient over B
            CTensor(optional), the gradient over C
        """
        _A, _B, C = self.inputs
        # y = alpha * A  * B  => da = alpha * dy * BT
        # y = alpha * A  * BT => da = alpha * dy * B
        # y = alpha * AT * B  => da = alpha * B * dyT = alpha * (dy * BT)T
        # y = alpha * AT * BT => da = alpha * BT * dyT = alpha * (dy * B)T
        da = singa.MultFloat(singa.Mult(dy, singa.DefaultTranspose(_B)),
                             self.alpha)
        if self.transA:
            da = singa.DefaultTranspose(da)

        # y = alpha * A  * B  => db = alpha * AT * dy
        # y = alpha * AT * B  => db = alpha * A * dy
        # y = alpha * A  * BT => db = alpha * dyT * A = alpha * (AT * dy)T
        # y = alpha * AT * BT => db = alpha * dyT * AT = alpha * (A * dy)T
        db = singa.MultFloat(singa.Mult(singa.DefaultTranspose(_A), dy),
                             self.alpha)
        if self.transB:
            db = singa.DefaultTranspose(db)
        if C:
            dc = back_broadcast(dy.shape(), C.shape(),
                                singa.MultFloat(dy, self.beta))
            return da, db, dc
        else:
            return da, db


def gemm(A, B, C=None, alpha=1.0, beta=1.0, transA=0, transB=0):
    """
    Init a General Matrix multiplication(Gemm) operator. Compute `Y = alpha * 
    A' * B' + beta * C`, where input tensor A has shape (M, K) or (K, M), input 
    tensor B has shape (K, N) or (N, K), input tensor C is broadcastable to 
    shape (M, N), and output tensor Y has shape (M, N).
    `A' = transpose(A)` if transA else A
    `B' = transpose(B)` if transB else B
    Args:
        A (Tensor): The shape of A should be (M, K) if transA is 0, or 
            (K, M) if transA is non-zero.
        B (Tensor): The shape of B should be (K, N) if transB is 0, or 
            (N, K) if transB is non-zero.
        C (Tensor): (optional), Optional input tensor C. If not specified, 
            the computation is done as if C is a scalar 0. The shape of C 
            should be unidirectional broadcastable to (M, N).
        alpha (float): Scalar multiplier for the product of input tensors A * B.
        beta (float): Scalar multiplier for input tensor C.
        ransA (int): Whether A should be transposed
        transB (int): Whether B should be transposed
    Returns: 
        Tensor, the output
    """
    return Gemm(alpha, beta, transA, transB)(A, B, C)[0]


class GlobalAveragePool(Operation):
    """
    Init a GlobalAveragePool operator
    """

    def __init__(self, data_format='channels_first'):
        """
        Args:
            data_format (string): A string, we support two formats: 
                channels_last and channels_first, default is channels_first.
                channels_first means the format of input is (N x C x H x W)
                channels_last means the format of input is (N x H x W x C)
        """
        super(GlobalAveragePool, self).__init__()
        self.data_format = data_format

    def forward(self, x):
        """
        forward propogation of GlobalAveragePool
        Args:
            x (CTensor): the input tensor
        Returns: 
            CTensor, the output
        """
        if training:
            self.mask = singa.Tensor(x.shape(), x.device())

        shape = list(x.shape())

        # (N x C x H x W) for channels_first
        if self.data_format == 'channels_first':
            axes = tuple(i for i in range(2, len(shape)))
            self.shape_divisor = 1 / np.prod(shape[2:])
        else:  # (N x H x W x C) for channels_last
            axes = tuple(i for i in range(1, len(shape) - 1))
            self.shape_divisor = 1 / np.prod(shape[1:-1])

        # output shape
        # (N x C x 1 x 1) for channels_first
        # (N x 1 x 1 x C) for channels_last
        for i in axes:
            shape[i] = 1

        x = tensor.from_raw_tensor(x)
        x = tensor.sum(x, axis=axes)
        x = tensor.reshape(x, shape)
        return singa.MultFloat(x.data, self.shape_divisor)

    def backward(self, dy):
        """
        backward propogation of GlobalAveragePool
        Args:
            dy (CTensor): the gradient tensor from upper operations
        Returns: 
            CTensor, the gradient over input
        """
        self.mask.SetFloatValue(self.shape_divisor)
        return singa.__mul__(self.mask, dy)


def globalaveragepool(x, data_format='channels_first'):
    """
    GlobalAveragePool operator
    Args:
        x (Tensor): the input tensor
        data_format (string): A string, we support two formats: 
            channels_last and channels_first, default is channels_first.
            channels_first means the format of input is (N x C x H x W)
            channels_last means the format of input is (N x H x W x C)
    Returns: 
        Tensor, the output
    """
    return GlobalAveragePool(data_format)(x)[0]


class ConstantOfShape(Operation):
    """
    Init a ConstantOfShape, generate a tensor with given value and shape.
    """

    def __init__(self, value=0.):
        """
        Args:
            value (float): (Optional) The value of the output elements. Should 
                be a one-element value. If not specified, it defaults to 0 and 
                datatype float32
        """
        super(ConstantOfShape, self).__init__()
        self.value = value

    def forward(self, x):
        """
        forward of ConstantOfShape
        Args:
            x: CTensor, 1D tensor. The shape of the expected output tensor. 
                All values must be >= 0.
        Returns:
            the output CTensor. If attribute 'value' is specified, the value 
                and datatype of the output tensor is taken from 'value'. If 
                attribute 'value' is not specified, the value in the output 
                defaults to 0, and the datatype defaults to float32.
        """
        x_shape = tensor.to_numpy(tensor.from_raw_tensor(x)).astype(
            np.int64).tolist()
        assert np.min(x_shape) >= 0, ('shape cannot be negative')
        x = CTensor(x_shape, x.device())
        x.SetFloatValue(self.value)
        return x

    def backward(self, dy):
        """
        backward of ConstantOfShape
        Args:
            dy (CTensor): gradient tensor.
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no gradient for backward function')


def constant_of_shape(x, value=0):
    """
    Init a ConstantOfShape, generate a tensor with given value and shape.
    Args:
        x: Tensor, 1D tensor. The shape of the expected output tensor. 
            All values must be >= 0.
        value (float): (Optional) The value of the output elements. Should 
            be a one-element value. If not specified, it defaults to 0 and 
            datatype float32
    Returns:
        the output Tensor. If attribute 'value' is specified, the value 
            and datatype of the output tensor is taken from 'value'. If 
            attribute 'value' is not specified, the value in the output 
            defaults to 0, and the datatype defaults to float32.
    """
    return ConstantOfShape(value)(x)[0]


class Dropout(Operation):
    """
    Init a Dropout, which scales the masked input data by the following equation:
    `output = scale * data * mask`, `scale = 1. / (1. - ratio)`.
    """

    def __init__(self, ratio=0.5):
        """
        Args:
            ratio (float): the ratio of random dropout, with value in [0, 1).
        """
        super(Dropout, self).__init__()
        self.ratio = ratio

    def forward(self, x):
        """
        forward of Dropout
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        if training:
            self.scale = 1 / 1 - self.ratio
            self.mask = singa.Tensor(list(x.shape()), x.device())
            singa.Bernoulli(1 - self.ratio, self.mask)
            x = singa.MultFloat(singa.__mul__(self.mask, x), self.scale)
        return x

    def backward(self, dy):
        """
        backward of Dropout
        Args:
            dy (CTensor): gradient tensor.
        Returns:
            the gradient tensor over input tensor.
        """
        if training:
            dy = singa.MultFloat(singa.__mul__(self.mask, dy), self.scale)
        return dy


def dropout(x, ratio=0.5):
    """
    Init a Dropout, which scales the masked input data by the following 
    equation: `output = scale * data * mask`, `scale = 1. / (1. - ratio)`.
    Args:
        x (Tensor): input tensor.
        ratio (float): the ratio of random dropout, with value in [0, 1).
    Returns:
        the output Tensor.
    """
    return Dropout(ratio)(x)[0]


class ReduceSum(Operation):
    """
    Init a ReduceSum, computes the sum of the input tensor's element along 
    the provided axes.
    """

    def __init__(self, axes=None, keepdims=1):
        """
        Args:
            axes (list of int): A list of integers, along which to reduce. 
                Accepted range is [-r, r-1] where r = rank(data). The default 
                is None, which reduces over all the dimensions of the input tensor.
            keepdims (int): Keep the reduced dimension or not, default 1 mean 
                keep reduced dimension.
        """
        super(ReduceSum, self).__init__()
        self.axes = axes
        self.keepdims = keepdims

    def forward(self, x):
        """
        forward of ReduceSum
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        _x = tensor.from_raw_tensor(x)
        x_shape = list(_x.shape)
        # handle the special axes
        if self.axes is None:
            self.axes = [i for i in range(len(x_shape))]  # axes = None
        else:
            self.axes = [i if i >= 0 else len(x_shape) + i for i in self.axes
                        ]  # axes has negative
        self.axes.sort(reverse=True)
        for axis in self.axes:
            _x = tensor.sum(_x, axis)
            x_shape[axis] = 1
        if self.keepdims == 1:
            _x = tensor.reshape(_x, x_shape)
        self.cache = (x_shape, x)
        return _x.data

    def backward(self, dy):
        """
        backward of ReduceSum
        Args:
            dy (CTensor): gradient tensor.
        Returns:
            the gradient tensor over input tensor.
        """
        x_shape, x = self.cache
        dy = singa.Reshape(dy, x_shape)
        scale = np.prod(x_shape) / np.prod(x.shape())
        mask = singa.Tensor(list(x.shape()), x.device())
        mask.SetFloatValue(scale)
        dy = singa.__mul__(mask, dy)
        return dy


def reduce_sum(x, axes=None, keepdims=1):
    """
    Init a ReduceSum, computes the sum of the input tensor's element along 
    the provided axes.
    Args:
        x (Tensor): input tensor.
        axes (list of int): A list of integers, along which to reduce. 
            Accepted range is [-r, r-1] where r = rank(data). The default 
            is None, which reduces over all the dimensions of the input tensor.
        keepdims (int): Keep the reduced dimension or not, default 1 mean 
            keep reduced dimension.
    Returns:
        the output Tensor.
    """
    return ReduceSum(axes, keepdims)(x)[0]


class ReduceMean(Operation):
    """
    Init a ReduceMean, computes the mean of the input tensor's element along 
    the provided axes.
    """

    def __init__(self, axes=None, keepdims=1):
        """
        Args:
            axes (list of int): A list of integers, along which to reduce. 
                Accepted range is [-r, r-1] where r = rank(data). The default 
                is None, which reduces over all the dimensions of the input tensor.
            keepdims (int): Keep the reduced dimension or not, default 1 mean 
                keep reduced dimension.
        """
        super(ReduceMean, self).__init__()
        self.axes = axes
        self.keepdims = keepdims

    def forward(self, x):
        """
        forward of ReduceMean
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        _x = tensor.from_raw_tensor(x)
        x_shape = list(_x.shape)
        # handle the special axes
        if self.axes is None:
            self.axes = [i for i in range(len(x_shape))]  # axes = None
        else:
            self.axes = [i if i >= 0 else len(x_shape) + i for i in self.axes
                        ]  # axes has negative
        self.axes.sort(reverse=True)
        for axis in self.axes:
            _x = tensor.sum(_x, axis)
            x_shape[axis] = 1
        if self.keepdims == 1:
            _x = tensor.reshape(_x, x_shape)
        self.cache = (x_shape, x)
        scale = np.prod(x_shape) / np.prod(x.shape())
        _x = singa.MultFloat(_x.data, scale)
        return _x

    def backward(self, dy):
        """
        backward of ReduceMean
        Args:
            dy (CTensor): gradient tensor.
        Returns:
            the gradient tensor over input tensor.
        """
        x_shape, x = self.cache
        dy = singa.Reshape(dy, x_shape)
        mask = singa.Tensor(list(x.shape()), x.device())
        mask.SetFloatValue(1.0)
        dy = singa.__mul__(mask, dy)
        return dy


def reduce_mean(x, axes=None, keepdims=1):
    """
    Init a ReduceMean, computes the mean of the input tensor's element along 
    the provided axes.
    Args:
        x (Tensor): input tensor.
        axes (list of int): A list of integers, along which to reduce. 
            Accepted range is [-r, r-1] where r = rank(data). The default 
            is None, which reduces over all the dimensions of the input tensor.
        keepdims (int): Keep the reduced dimension or not, default 1 mean 
            keep reduced dimension.
    Returns:
        the output Tensor.
    """
    return ReduceMean(axes, keepdims)(x)[0]


class Slice(Operation):
    """
    Init a Slice, Produces a slice of the input tensor along multiple axes. 
    Similar to numpy: https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html
    """

    def __init__(self, starts, ends, axes=None, steps=None):
        """
        Args:
            starts (list of int): starting indices of corresponding axis
            ends (list of int): ending indices of corresponding axis
            axes (list of int): axes that `starts` and `ends` apply to. 
                Negative value means counting dimensions from the back. 
                Accepted range is [-r, r-1] where r = rank(data).
            steps (list of int): slice step of corresponding axis in `axes`. 
                Negative value means slicing backward. 'steps' cannot be 0. 
                Defaults to 1.
        """
        super(Slice, self).__init__()
        self.starts = starts
        self.ends = ends
        self.axes = axes
        self.steps = steps

    def forward(self, x):
        """
        forward of Slice
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        x_shape = list(x.shape())
        # handle the special axes
        if self.axes is None:
            self.axes = [i for i in range(len(x_shape))]  # axes = None
        else:
            self.axes = [i if i >= 0 else len(x_shape) + i for i in self.axes
                        ]  # axes has negative
        self.cache = []
        # handle the special steps
        if self.steps is None:
            self.steps = [1] * len(x_shape)  # steps = None
        for idx, axis in enumerate(self.axes):
            start, end, step = self.starts[idx], self.ends[idx], self.steps[idx]
            if end > x_shape[axis]:
                end = x_shape[axis]
            self.cache.append((axis, x_shape[axis], start, end, step))
            xs = []
            for step_idx in range(x_shape[axis])[start:end:step]:
                xs.append(singa.SliceOn(x, step_idx, step_idx + 1, axis))
            assert len(xs) > 0, "Cannot support empty tensor"
            x = singa.VecTensor(xs)
            x = singa.ConcatOn(x, axis)
        return x

    def backward(self, dy):
        """
        backward of Slice
        Args:
            dy (CTensor): gradient tensor.
        Returns:
            the gradient tensor over input tensor.
        """
        for axis, shape, start, end, step in self.cache[::-1]:
            data_idxes = tuple(range(shape)[start:end:step])
            dys = []
            data_idx = 0
            for step_idx in range(shape):
                if step_idx in data_idxes:
                    tmp_tensor = singa.SliceOn(dy, data_idx, data_idx + 1, axis)
                    data_idx += 1
                else:
                    tmp_shape = list(dy.shape())
                    tmp_shape[axis] = 1
                    tmp_tensor = singa.Tensor(tmp_shape, dy.device())
                    tmp_tensor.SetFloatValue(0.)
                dys.append(tmp_tensor)
            dys = singa.VecTensor(dys)
            dy = singa.ConcatOn(dys, axis)
        return dy


def slice(x, starts, ends, axes=None, steps=None):
    """
    Init a Slice, Produces a slice of the input tensor along multiple axes. 
    Similar to numpy: https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html
    Args:
        x (Tensor): input tensor.
        starts (list of int): starting indices of corresponding axis
        ends (list of int): ending indices of corresponding axis
        axes (list of int): axes that `starts` and `ends` apply to. 
            Negative value means counting dimensions from the back. 
            Accepted range is [-r, r-1] where r = rank(data).
        steps (list of int): slice step of corresponding axis in `axes`. 
            Negative value means slicing backward. 'steps' cannot be 0. 
            Defaults to 1.
    Returns:
        the output Tensor.
    """
    return Slice(starts, ends, axes, steps)(x)[0]


class Ceil(Operation):
    """
    Ceil takes one input data (Tensor) and produces one output data (Tensor) 
    where the ceil is, `y = ceil(x)`, is applied to the tensor elementwise.
    """

    def __init__(self):
        super(Ceil, self).__init__()

    def forward(self, x):
        """
        forward of Ceil
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        return singa.Ceil(x)

    def backward(self, dy):
        """
        backward of Ceil
        Args:
            dy (CTensor): gradient tensor.
        Returns:
            the gradient tensor over input tensor.
        """
        dy = singa.Tensor(dy.shape(), dy.device())
        dy.SetFloatValue(0.)
        return dy


def ceil(x):
    """
    Ceil takes one input data (Tensor) and produces one output data (Tensor) 
    where the ceil is, `y = ceil(x)`, is applied to the tensor elementwise.
    Args:
        x (Tensor): input tensor.
    Returns:
        the output Tensor.
    """
    return Ceil()(x)[0]


class Split(Operation):
    """
    Init a Split, Split a tensor into a list of tensors, along the specified 
    'axis'. 
    """

    def __init__(self, axis, parts, num_output=None):
        """
        Args:
            axis (int): which axis to split on. A negative value means 
                counting dimensions from the back. Accepted range is 
                [-rank, rank-1] where r = rank(input).
            parts (list of int): length of each output, which can be specified 
                using argument 'parts'. Otherwise, the tensor is parts to equal 
                sized parts.
            num_output (bool): once parts is none, the tensor is split to equal 
                sized parts for each output.
        """
        super(Split, self).__init__()
        self.axis = axis
        self.parts = parts
        self.num_output = num_output
        if self.parts is None:
            assert self.num_output is not None, "For (parts, num_output), it at least requires one."

    def forward(self, x):
        """
        forward of Split
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        x_shape = list(x.shape())
        self.axis = self.axis % len(x_shape)
        if self.parts is None:
            self.parts = [x_shape[self.axis] // self.num_output
                         ] * self.num_output
        xs = []
        _s = 0
        for _l in self.parts:
            xs.append(singa.SliceOn(x, _s, _s + _l, self.axis))
            _s += _l
        return tuple(xs)

    def backward(self, *dys):
        """
        backward of Split
        Args:
            dys: list of CTensor, gradient tensor.
        Returns:
            the gradient tensor over input tensor.
        """
        dys = singa.VecTensor(dys)
        dy = singa.ConcatOn(dys, self.axis)
        return dy


def split(x, axis, parts, num_output=None):
    """
    Init a Split, Split a tensor into a list of tensors, along the specified 
    'axis'. 
    Args:
        x (Tensor): input tensor.
        axis (int): which axis to split on. A negative value means 
            counting dimensions from the back. Accepted range is 
            [-rank, rank-1] where r = rank(input).
        parts (list of int): length of each output, which can be specified 
            using argument 'parts'. Otherwise, the tensor is parts to equal 
            sized parts.
        num_output (bool): once parts is none, the tensor is split to equal 
            sized parts for each output.
    Returns:
        the output Tensor.
    """
    return Split(axis, parts, num_output)(x)


class Gather(Operation):
    """
    Init a Gather, Given data tensor of rank r >= 1, and indices tensor of 
    rank q, gather entries of the axis dimension of data (by default outer-most 
    one as axis=0) indexed by indices, and concatenates them in an output tensor of rank `q + (r - 1)`.
    """

    def __init__(self, axis, indices):
        """
        Args:
            axis (int): which axis to slice on. A negative value means counting 
                dimensions from the back. Accepted range is [-rank, rank-1] 
                where r = rank(input).
            indices (list of int): entries of the axis dimension of data.
        """
        super(Gather, self).__init__()
        self.axis = axis
        self.indices = indices

    def forward(self, x):
        """
        forward of Gather
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        self.x_shape = list(x.shape())
        self.axis = self.axis % len(self.x_shape)  # handle the negative value
        _shape = self.x_shape[self.axis]
        xs = []
        for indice in self.indices:
            # each indice is a sub-indice
            if isinstance(indice, tuple) or isinstance(indice, list):
                sub_xs = []
                for idx in indice:
                    idx = idx % _shape
                    tmp_tensor = singa.SliceOn(x, idx, idx + 1, self.axis)
                    sub_xs.append(tmp_tensor)
                sub_xs = singa.VecTensor(sub_xs)
                tmp_tensor = singa.ConcatOn(sub_xs, self.axis)
                _slice_shape = list(tmp_tensor.shape())
                _slice_shape.insert(self.axis, 1)  # add a new axis to concat
                tmp_tensor = singa.Reshape(tmp_tensor, _slice_shape)
            else:
                indice = indice % _shape
                tmp_tensor = singa.SliceOn(x, indice, indice + 1, self.axis)
            xs.append(tmp_tensor)
        xs = singa.VecTensor(xs)
        return singa.ConcatOn(xs, self.axis)

    def backward(self, dy):
        """
        backward of Gather
        Args:
            dy (CTensor): gradient tensor.
        Returns:
            the gradient tensor over input tensor.
        """
        _shape = self.x_shape[self.axis]

        def construct_dx(dy, axis, indices, _shape):
            dys = []
            data_idx = 0
            data_idxes = tuple(indices)
            for step_idx in range(_shape):
                if step_idx in data_idxes:
                    tmp_tensor = singa.SliceOn(dy, data_idx, data_idx + 1, axis)
                    data_idx += 1
                else:
                    tmp_shape = list(dy.shape())
                    tmp_shape[axis] = 1
                    tmp_tensor = singa.Tensor(tmp_shape, dy.device())
                    tmp_tensor.SetFloatValue(0.)
                dys.append(tmp_tensor)
            dys = singa.VecTensor(dys)
            dy = singa.ConcatOn(dys, axis)
            return dy

        if isinstance(self.indices[0], tuple) or isinstance(
                self.indices[0], list):
            dx = singa.Tensor(self.x_shape, dy.device())
            dx.SetFloatValue(0.)
            for data_idx in range(len(self.indices)):
                # get a piece of the dy and remove its new axis added at forward
                tmp_tensor = singa.SliceOn(dy, data_idx, data_idx + 1,
                                           self.axis)
                _slice_shape = list(tmp_tensor.shape())
                del _slice_shape[self.axis]
                tmp_tensor = singa.Reshape(tmp_tensor, _slice_shape)
                # construct dx and sum them together
                tmp_tensor = construct_dx(tmp_tensor, self.axis,
                                          self.indices[data_idx],
                                          self.x_shape[self.axis])
                dx = singa.__add__(dx, tmp_tensor)
            return dx
        else:
            return construct_dx(dy, self.axis, self.indices, _shape)


def gather(x, axis, indices):
    """
    Init a Gather, Given data tensor of rank r >= 1, and indices tensor of 
    rank q, gather entries of the axis dimension of data (by default outer-most 
    one as axis=0) indexed by indices, and concatenates them in an output tensor of rank `q + (r - 1)`.
    Args:
        x (Tensor): input tensor.
        axis (int): which axis to slice on. A negative value means counting 
            dimensions from the back. Accepted range is [-rank, rank-1] 
            where r = rank(input).
        indices (list of int): entries of the axis dimension of data.
    Returns:
        the output Tensor.
    """
    return Gather(axis, indices)(x)[0]


class Tile(Operation):
    """
    Init a Tile, Constructs a tensor by tiling a given tensor. This is the same 
    as function tile in Numpy: https://docs.scipy.org/doc/numpy/reference/generated/numpy.tile.html
    """

    def __init__(self, repeats):
        """
        Args:
            repeats (list of int): 1D int matrix of the same length as input's 
                dimension number, includes numbers of repeated copies along 
                input's dimensions.
        """
        super(Tile, self).__init__()
        self.repeats = [repeats] if isinstance(repeats, int) else repeats

    def forward(self, x):
        """
        forward of Tile
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        self.x_shape = list(x.shape())
        # add new axis from head
        if len(self.x_shape) < len(self.repeats):
            append_len = len(self.repeats) - len(self.x_shape)
            new_shape = [1] * append_len + self.x_shape
            x = singa.Reshape(x, new_shape)
        for axis, rp in enumerate(self.repeats):
            if rp == 1:
                continue
            xs = []
            for idx in range(rp):
                xs.append(x.Clone())
            xs = singa.VecTensor(xs)
            x = singa.ConcatOn(xs, axis)
        return x

    def backward(self, dy):
        """
        backward of Tile
        Args:
            dy (CTensor): gradient tensor.
        Returns:
            the gradient tensor over input tensor.
        """
        for axis, rp in enumerate(self.repeats):
            if rp == 1:
                continue
            _slice_shape = list(dy.shape())
            ori_len = _slice_shape[axis] // rp
            _slice_shape[axis] = ori_len
            _dy = singa.Tensor(_slice_shape, dy.device())
            _dy.SetFloatValue(0.)

            for idx in range(rp):
                tmp_tensor = singa.SliceOn(dy, ori_len * idx,
                                           ori_len * (idx + 1), axis)
                _dy = singa.__add__(_dy, tmp_tensor)
            dy = _dy
        # remove the new axis we added at forward
        if len(self.x_shape) < len(self.repeats):
            dy = singa.Reshape(dy, self.x_shape)
        return dy


def tile(x, repeats):
    """
    Init a Tile, Constructs a tensor by tiling a given tensor. This is the same 
    as function tile in Numpy: https://docs.scipy.org/doc/numpy/reference/generated/numpy.tile.html
    Args:
        x (Tensor): input tensor.
        repeats (list of int): 1D int matrix of the same length as input's 
            dimension number, includes numbers of repeated copies along 
            input's dimensions.
    Returns:
        the output Tensor.
    """
    return Tile(repeats)(x)[0]


class NonZero(Operation):
    """
    Init a NonZero, Constructs a tensor by tiling a given tensor. This is the same 
    as function tile in Numpy: https://docs.scipy.org/doc/numpy/reference/generated/numpy.tile.html
    """

    def __init__(self):
        super(NonZero, self).__init__()

    def forward(self, x):
        """
        forward of NonZero
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        y = tensor.to_numpy(tensor.from_raw_tensor(x))
        y = np.array((np.nonzero(y))).astype(np.int32)
        y = tensor.from_numpy(y)
        y.to_device(x.device())
        return y.data

    def backward(self, dy):
        """
        backward of NonZero
        Args:
            dy (CTensor): gradient tensor.
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no gradient for backward function')


def nonzero(x):
    """
    Init a NonZero, Constructs a tensor by tiling a given tensor. This is the same 
    as function tile in Numpy: https://docs.scipy.org/doc/numpy/reference/generated/numpy.tile.html
    Args:
        x (Tensor): input tensor.
    Returns:
        the output Tensor.
    """
    return NonZero()(x)[0]


class Cast(Operation):
    """
    The operator casts the elements of a given input tensor to a data type 
    specified by the 'to' argument and returns an output tensor of the same 
    size in the converted type.
    """

    def __init__(self, to):
        """
        Args:
            to (int): data type, float32 = 0; int = 2.
        """
        super(Cast, self).__init__()
        self.to = to

    def forward(self, x):
        """
        forward of Cast
        Args:
            x (CTensor): input tensor.
        Returns:
            the output CTensor.
        """
        if x.data_type() != self.to:
            x = x.AsType(self.to)
        return x

    def backward(self, dy):
        """
        backward of Cast
        Args:f
            dy (CTensor), gradient tensor.
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no gradient for backward function')


def cast(x, to):
    """
    The operator casts the elements of a given input tensor to a data type 
    specified by the 'to' argument and returns an output tensor of the same 
    size in the converted type.
    Args:
        x (Tensor): input tensor.
        to (int): data type, float32 = 0; int = 2.
    Returns:
        the output Tensor.
    """
    return Cast(to)(x)[0]


class OneHot(Operation):
    """
    Produces a one-hot tensor based on inputs. 
    """

    def __init__(self, axis, depth, values):
        """
        Args:
            axis (int): Axis along which one-hot representation in added. 
                Default: axis=-1. axis=-1 means that the additional dimension 
                will be inserted as the innermost/last dimension in the output 
                tensor.
            depth (int): Scalar specifying the number of classes in one-hot 
                tensor. This is also the size of the one-hot dimension 
                (specified by 'axis' attribute) added on in the output tensor. 
                The values in the 'indices' input tensor are expected to be in 
                the range [-depth, depth-1].
            values (float): Rank 1 tensor containing exactly two elements, in 
                the format [off_value, on_value], where 'on_value' is the 
                value used for filling locations specified in 'indices' input 
                tensor, 
        """
        super(OneHot, self).__init__()
        self.axis = axis
        self.depth = depth
        self.values = values

    def forward(self, indices):
        """
        forward of OneHot, we borrow this function from onnx
        Args:
            indices (CTensor): Scalar specifying the number of classes in 
                one-hot tensor. The values in the 'indices' input tensor are 
                expected to be in the range [-depth, depth-1]. 
        Returns:
            the output CTensor.
        """
        values = tensor.to_numpy(tensor.from_raw_tensor(indices))
        rank = len(values.shape)
        depth_range = np.arange(self.depth)
        if self.axis < 0:
            self.axis += (rank + 1)
        ls = values.shape[0:self.axis]
        rs = values.shape[self.axis:rank]
        targets = np.reshape(depth_range, (1,) * len(ls) + depth_range.shape +
                             (1,) * len(rs))
        values = np.reshape(np.mod(values, self.depth), ls + (1,) + rs)
        np_tensor = np.asarray(targets == values, dtype=np.float32)
        np_tensor = np_tensor * (self.values[1] -
                                 self.values[0]) + self.values[0]
        tmp_tensor = tensor.from_numpy(np_tensor)
        tmp_tensor.to_device(indices.device())
        return tmp_tensor.data

    def backward(self, dy):
        """
        backward of OneHot
        Args:
            dy (CTensor):gradient tensor.
        Raises:
            AssertionError: no backward function for this operator
        """
        assert False, ('no gradient for backward function')


def onehot(axis, indices, depth, values):
    """
    Produces a one-hot tensor based on inputs. 
    Args:
        axis (int): Axis along which one-hot representation in added. 
            Default: axis=-1. axis=-1 means that the additional dimension 
            will be inserted as the innermost/last dimension in the output 
            tensor.
        indices (Tensor): Scalar specifying the number of classes in 
            one-hot tensor. The values in the 'indices' input tensor are 
            expected to be in the range [-depth, depth-1]. 
        depth (int): Scalar specifying the number of classes in one-hot 
            tensor. This is also the size of the one-hot dimension 
            (specified by 'axis' attribute) added on in the output tensor. 
            The values in the 'indices' input tensor are expected to be in 
            the range [-depth, depth-1].
        values (float): Rank 1 tensor containing exactly two elements, in 
            the format [off_value, on_value], where 'on_value' is the 
            value used for filling locations specified in 'indices' input 
            tensor, 
    Returns:
        the output Tensor.
    """
    return OneHot(axis, depth, values)(indices)[0]