Merge pull request #872 from nudles/dev
Prepare for v3.2.0 release
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1d8201f..a722ec9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,10 +29,10 @@
#string(REGEX REPLACE "^[0-9]+\\.[0-9]+\\.([0-9]+).*" "\\1" VERSION_PATCH "${VERSION}")
-SET(PACKAGE_VERSION 3.1.0) # ${VERSION})
-SET(VERSION 3.1.0)
+SET(PACKAGE_VERSION 3.2.0) # ${VERSION})
+SET(VERSION 3.2.0)
SET(SINGA_MAJOR_VERSION 3)
-SET(SINGA_MINOR_VERSION 1)
+SET(SINGA_MINOR_VERSION 2)
SET(SINGA_PATCH_VERSION 0)
#SET(SINGA_MAJOR_VERSION ${VERSION_MAJOR}) # 0 -
#SET(SINGA_MINOR_VERSION ${VERSION_MINOR}) # 0 - 9
diff --git a/python/singa/tensor.py b/python/singa/tensor.py
index e9e9ae7..963ad1a 100755
--- a/python/singa/tensor.py
+++ b/python/singa/tensor.py
@@ -674,72 +674,65 @@
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__add__(self.data, rhs.data))
else:
- return _call_singa_func(singa.AddFloat, self.data, rhs)
-
+ return _call_singa_func(singa.AddFloat, self.data, float(rhs))
+
def __sub__(self, rhs):
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__sub__(self.data, rhs.data))
else:
- return _call_singa_func(singa.SubFloat, self.data, rhs)
-
+ return _call_singa_func(singa.SubFloat, self.data, float(rhs))
+
def __mul__(self, rhs):
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__mul__(self.data, rhs.data))
else:
- return _call_singa_func(singa.MultFloat, self.data, rhs)
-
+ return _call_singa_func(singa.MultFloat, self.data, float(rhs))
+
def __div__(self, rhs):
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__div__(self.data, rhs.data))
else:
- return _call_singa_func(singa.DivFloat, self.data, rhs)
-
+ return _call_singa_func(singa.DivFloat, self.data, float(rhs))
+
def __truediv__(self, rhs):
- if isinstance(rhs, Tensor):
- return from_raw_tensor(singa.__div__(self.data, rhs.data))
- else:
- return _call_singa_func(singa.DivFloat, self.data, rhs)
-
+ return self.__div__(rhs)
+
def __floordiv__(self, rhs):
- if isinstance(rhs, Tensor):
- tmp = from_raw_tensor(singa.__div__(self.data, rhs.data))
- return _call_singa_func(singa.Floor, tmp.data)
- else:
- tmp = _call_singa_func(singa.DivFloat, self.data, rhs)
- return _call_singa_func(singa.Floor, tmp.data)
+ tmp = self.__div__(rhs)
+ return _call_singa_func(singa.Floor, tmp.data)
def __lt__(self, rhs):
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__lt__(self.data, rhs.data))
else:
- return _call_singa_func(singa.LTFloat, self.data, rhs)
-
+ return _call_singa_func(singa.LTFloat, self.data, float(rhs))
+
def __le__(self, rhs):
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__le__(self.data, rhs.data))
else:
- return _call_singa_func(singa.LEFloat, self.data, rhs)
-
+ return _call_singa_func(singa.LEFloat, self.data, float(rhs))
+
def __gt__(self, rhs):
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__gt__(self.data, rhs.data))
else:
- return _call_singa_func(singa.GTFloat, self.data, rhs)
-
+ return _call_singa_func(singa.GTFloat, self.data, float(rhs))
+
def __ge__(self, rhs):
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__ge__(self.data, rhs.data))
else:
- return _call_singa_func(singa.GEFloat, self.data, rhs)
-
+ return _call_singa_func(singa.GEFloat, self.data, float(rhs))
+
def __eq__(self, rhs):
if isinstance(rhs, Tensor):
return from_raw_tensor(singa.__eq__(self.data, rhs.data))
elif rhs is None:
return False
else:
- return _call_singa_func(singa.EQFloat, self.data, rhs)
-
+ return _call_singa_func(singa.EQFloat, self.data, float(rhs))
+
def __radd__(self, lhs):
lhs = float(lhs)
one = Tensor(self.shape, self.device, self.dtype)
diff --git a/setup.py b/setup.py
index f7a3f19..4e9a644 100644
--- a/setup.py
+++ b/setup.py
@@ -83,7 +83,7 @@
from datetime import date
# stable version
-VERSION = '3.1.0.rc1'
+VERSION = '3.2.0'
# get the git hash
# git_hash = subprocess.check_output(["git", "describe"]).strip().split('-')[-1][1:]
# comment the next line to build wheel for stable version
@@ -267,9 +267,9 @@
extra_compile_args = {'gcc': get_cpp_flags()}
if with_cuda:
- cuda9_gencode = (' -gencode arch=compute_35,code=sm_35'
- ' -gencode arch=compute_50,code=sm_50'
- ' -gencode arch=compute_60,code=sm_60'
+ # compute_35 and compute_50 are removed because 1. they do not support half float;
+ # 2. google colab's GPU has been updated from K80 (compute_35) to T4 (compute_75).
+ cuda9_gencode = (' -gencode arch=compute_60,code=sm_60'
' -gencode arch=compute_70,code=sm_70')
cuda10_gencode = ' -gencode arch=compute_75,code=sm_75'
cuda11_gencode = ' -gencode arch=compute_80,code=sm_80'
@@ -422,6 +422,7 @@
'numpy >=1.16,<2.0', #1.16
'onnx==1.6',
'deprecated',
+ 'pytest',
'unittest-xml-reporting',
'future',
'pillow',
diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc
index 08e5d41..46b82aa 100644
--- a/src/core/tensor/tensor.cc
+++ b/src/core/tensor/tensor.cc
@@ -1037,33 +1037,59 @@
}); \
} while (0)
-#define GenBinaryTensorFn(op, fn) \
- Tensor op(const Tensor &lhs, const Tensor &rhs) { \
- if (lhs.shape() != rhs.shape()) { \
- auto lhs_ = Broadcast(lhs, rhs.shape()); \
- auto rhs_ = Broadcast(rhs, lhs.shape()); \
- Tensor ret(lhs_.shape(), lhs.device(), lhs.data_type()); \
- fn(lhs_, rhs_, &ret); \
- return ret; \
- } else { \
- Tensor ret(lhs.shape(), lhs.device(), lhs.data_type()); \
- fn(lhs, rhs, &ret); \
- return ret; \
- } \
- } \
- void fn(const Tensor &lhs, const Tensor &rhs, Tensor *ret) { \
- CHECK_EQ(lhs.device(), ret->device()); \
- CHECK_EQ(rhs.device(), ret->device()); \
- if (lhs.shape() != rhs.shape()) { \
- auto lhs_ = Broadcast(lhs, rhs.shape()); \
- auto rhs_ = Broadcast(rhs, lhs.shape()); \
- CHECK(lhs_.shape() == ret->shape()); \
- EltwiseBinaryTensorFn(fn, lhs_, rhs_, ret); \
- } else { \
- CHECK(lhs.shape() == ret->shape()); \
- EltwiseBinaryTensorFn(fn, lhs, rhs, ret); \
- } \
- } // namespace singa
+#define GenBinaryTensorFn(op, fn) \
+ Tensor op(const Tensor &lhs, const Tensor &rhs) { \
+ if (lhs.shape() != rhs.shape()) { \
+ if (lhs.data_type() == kFloat32 && rhs.data_type() == kFloat32) { \
+ auto lhs_ = Broadcast(lhs, rhs.shape()); \
+ auto rhs_ = Broadcast(rhs, lhs.shape()); \
+ Tensor ret(lhs_.shape(), lhs.device(), lhs.data_type()); \
+ fn(lhs_, rhs_, &ret); \
+ return ret; \
+ } else { \
+ /* lhs tensor and rhs tensor are not both in float, cast to float */\
+ Tensor tmp_lhs = lhs.Clone().AsType(kFloat32); \
+ Tensor tmp_rhs = rhs.Clone().AsType(kFloat32); \
+ tmp_lhs = Broadcast(tmp_lhs, tmp_rhs.shape()); \
+ tmp_rhs = Broadcast(tmp_rhs, tmp_lhs.shape()); \
+ Tensor ret(tmp_lhs.shape(), tmp_lhs.device(), tmp_lhs.data_type()); \
+ fn(tmp_lhs, tmp_rhs, &ret); \
+ /* if lhs and rhs are both int, cast back to int */ \
+ if (lhs.data_type() == kInt && rhs.data_type() == kInt) \
+ return ret.Clone().AsType(kInt); \
+ return ret; \
+ } \
+ } else { \
+ if (lhs.data_type() == kFloat32 && rhs.data_type() == kFloat32) { \
+ Tensor ret(lhs.shape(), lhs.device(), lhs.data_type()); \
+ fn(lhs, rhs, &ret); \
+ return ret; \
+ } else { \
+ /* lhs tensor and rhs tensor are not both in float, cast to float */\
+ Tensor tmp_lhs = lhs.Clone().AsType(kFloat32); \
+ Tensor tmp_rhs = rhs.Clone().AsType(kFloat32); \
+ Tensor ret(tmp_lhs.shape(), tmp_lhs.device(), tmp_lhs.data_type()); \
+ fn(tmp_lhs, tmp_rhs, &ret); \
+ /* if lhs and rhs are both int, cast back to int */ \
+ if (lhs.data_type() == kInt && rhs.data_type() == kInt) \
+ return ret.Clone().AsType(kInt); \
+ return ret; \
+ } \
+ } \
+ } \
+ void fn(const Tensor &lhs, const Tensor &rhs, Tensor *ret) { \
+ CHECK_EQ(lhs.device(), ret->device()); \
+ CHECK_EQ(rhs.device(), ret->device()); \
+ if (lhs.shape() != rhs.shape()) { \
+ auto lhs_ = Broadcast(lhs, rhs.shape()); \
+ auto rhs_ = Broadcast(rhs, lhs.shape()); \
+ CHECK(lhs_.shape() == ret->shape()); \
+ EltwiseBinaryTensorFn(fn, lhs_, rhs_, ret); \
+ } else { \
+ CHECK(lhs.shape() == ret->shape()); \
+ EltwiseBinaryTensorFn(fn, lhs, rhs, ret); \
+ } \
+ }
// boradcasting operations:
// https://github.com/onnx/onnx/blob/master/docs/Broadcasting.md
@@ -1093,12 +1119,29 @@
} while (0)
#define GenTensorScalarFn(op, fn) \
- template <typename SType> \
- Tensor op(const Tensor &in, const SType x) { \
- Tensor ret(in.shape(), in.device(), in.data_type()); \
- fn(in, x, &ret); \
- return ret; \
- } \
+ template <typename SType> \
+ Tensor op(const Tensor &in, const SType x) { \
+ if (in.data_type() == kFloat32 && std::is_same<SType, float>::value){ \
+ Tensor ret(in.shape(), in.device(), in.data_type()); \
+ fn(in, x, &ret); \
+ return ret; \
+ } else if (in.data_type() == kFloat32) { \
+ Tensor ret(in.shape(), in.device(), in.data_type()); \
+ float tmp_x = x; \
+ fn(in, tmp_x, &ret); \
+ return ret; \
+ } else { \
+ /* tensor and scalar are not both in float, cast to float */ \
+ Tensor tmp_in = in.Clone().AsType(kFloat32); \
+ float tmp_x = x; \
+ Tensor ret(tmp_in.shape(), tmp_in.device(), tmp_in.data_type()); \
+ fn(tmp_in, tmp_x, &ret); \
+ /* if tensor and scalar are both int, cast back to int */ \
+ if (in.data_type() == kInt && std::is_same<SType, int>::value) \
+ return ret.Clone().AsType(kInt); \
+ return ret; \
+ } \
+ } \
template <typename SType> \
void fn(const Tensor &in, const SType x, Tensor *ret) { \
EltwiseTensorScalarFn(fn, in, x, ret); \
diff --git a/src/core/tensor/tensor_math.h b/src/core/tensor/tensor_math.h
index 2e6a08a..c294c4e 100644
--- a/src/core/tensor/tensor_math.h
+++ b/src/core/tensor/tensor_math.h
@@ -55,6 +55,9 @@
/// 7. Use size_t for the number of elements, rows or columns.
/// 8. Use the same name for the Tensor and Tensor level math functions.
+#define LOG_FATAL(Op, DType, Lang) \
+ LOG(FATAL) << Op << " not Implemented for DType=" << typeid(DType).name() << " Lang=" << typeid(Lang).name()
+
const std::string vec2str(const std::vector<int> &vec) {
std::ostringstream vts;
if (!vec.empty()) {
@@ -83,62 +86,63 @@
/// out[i] = |in[i]|
template <typename DType, typename Lang>
void Abs(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Abs Not Implemented";
+ LOG_FATAL("Abs", DType, Lang);
}
template <typename DType, typename Lang>
void Erf(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Erf Not Implemented";
+ LOG_FATAL("Erf", DType, Lang);
}
template <typename DTypeSrc, typename DTypeDst, typename Lang>
void CastCopy(const Tensor *src, Tensor *dst, Context *ctx) {
- LOG(FATAL) << "CastCopy Not Implemented";
+ LOG(FATAL) << "CastCopy not Implemented for DTypeSrc=" << typeid(DTypeSrc).name()
+ << " DTypeDst=" << typeid(DTypeDst).name() << " Lang=" << typeid(Lang).name();
}
template <typename DType, typename Lang>
void Ceil(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Ceil Not Implemented";
+ LOG_FATAL("Ceil", DType, Lang);
}
template <typename DType, typename Lang>
void Floor(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Floor Not Implemented";
+ LOG_FATAL("Floor", DType, Lang);
}
template <typename DType, typename Lang>
void Round(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Round Not Implemented";
+ LOG_FATAL("Round", DType, Lang);
}
template <typename DType, typename Lang>
void RoundE(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Round Not Implemented";
+ LOG_FATAL("RoundE", DType, Lang);
}
/// out[i] = in[i] + x
template <typename DType, typename Lang>
void Add(const Tensor &in, const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Add Not Implemented";
+ LOG_FATAL("Add", DType, Lang);
}
/// out[i] = in1[i] + in2[i]
template <typename DType, typename Lang>
void Add(const Tensor &in1, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Add-Pair Not Implemented";
+ LOG_FATAL("Add-Pair", DType, Lang);
}
/// Clamp every element into [low, high]
/// if in[i]>high, then out[i]=high; if in[i]<low, then out[i]=low.
template <typename DType, typename Lang>
void Clamp(const DType low, const DType high, const Tensor &in, Tensor *out,
Context *ctx) {
- LOG(FATAL) << "Clamp Not Implemented";
+ LOG_FATAL("Clamp", DType, Lang);
}
/// out[i] = x / in[i]
template <typename DType, typename Lang>
void Div(const DType x, const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Div Not Implemented";
+ LOG_FATAL("Div", DType, Lang);
}
/// out[i] = in[i] / x
@@ -151,140 +155,140 @@
/// out[i] = in1[i] / in2[i]
template <typename DType, typename Lang>
void Div(const Tensor &in1, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Div-Pair Not Implemented";
+ LOG_FATAL("Div-Pair", DType, Lang);
}
/// out[i] = in[i] * x
template <typename DType, typename Lang>
void EltwiseMult(const Tensor &in, const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "EltwiseMult Not Implemented";
+ LOG_FATAL("EltwiseMult", DType, Lang);
}
/// out[i] = in1[i] * in2[i]
template <typename DType, typename Lang>
void EltwiseMult(const Tensor &in1, const Tensor &in2, Tensor *out,
Context *ctx) {
- LOG(FATAL) << "EltwiseMult-Pair Not Implemented";
+ LOG_FATAL("EltwiseMult-Pair", DType, Lang);
}
/// out[i]=(in2[i]>0)?in1[i]:0.f
template <typename DType, typename Lang>
void ReLUBackward(const Tensor &in1, const Tensor &in2, Tensor *out,
Context *ctx) {
- LOG(FATAL) << "ReLUBackward Not Implemented";
+ LOG_FATAL("ReLUBackward", DType, Lang);
}
/// Base is e, Neper number. out[i]=exp(in[i])
template <typename DType, typename Lang>
void Exp(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Exp Not Implemented";
+ LOG_FATAL("Exp", DType, Lang);
}
/// out[i]=(in[i]<=x)?1.f:0.f
template <typename DType, typename Lang>
void LE(const Tensor &in, const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "LE Not Implemented";
+ LOG_FATAL("LE", DType, Lang);
}
/// out[i]=(in1[i]<=in2[i])?1.f:0.f
template <typename DType, typename Lang>
void LE(const Tensor &in1, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Tensor-Tensor LE Not Implemented";
+ LOG_FATAL("Tensor <= Tensor", DType, Lang);
}
/// Natural logarithm, the base is e, Neper number out[i]=log(in[i]).
template <typename DType, typename Lang>
void Log(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Log Not Implemented";
+ LOG_FATAL("Log", DType, Lang);
}
/// out[i]=(in[i]<x)?1.f:0.f
template <typename DType, typename Lang>
void LT(const Tensor &in, const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "LT Not Implemented";
+ LOG_FATAL("LT", DType, Lang);
}
/// out[i]=(in1[i]<in2[i])?1.f:0.f
template <typename DType, typename Lang>
void LT(const Tensor &in1, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Tensor-Tensor LT Not Implemented";
+ LOG_FATAL("Tensor Tensor LT", DType, Lang);
}
/// out[i]=(in[i]>=x)?1.f:0.f
template <typename DType, typename Lang>
void GE(const Tensor &in, const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "GE Not Implemented";
+ LOG_FATAL("GE", DType, Lang);
}
/// out[i]=(in1[i]>=in2[i])?1.f:0.f
template <typename DType, typename Lang>
void GE(const Tensor &in1, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Tensor-Tensor GE Not Implemented";
+ LOG_FATAL("Tensor Tensor GE", DType, Lang);
}
/// out[i]=(in[i]>x)?1.f:0.f
template <typename DType, typename Lang>
void GT(const Tensor &in, const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "GT Not Implemented";
+ LOG_FATAL("GT", DType, Lang);
}
/// out[i]=(in[i]>in2[i])?1.f:0.f
template <typename DType, typename Lang>
void GT(const Tensor &in, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Tensor-Tensor GT Not Implemented";
+ LOG_FATAL("Tensor Tensor GT", DType, Lang);
}
/// out[i]=(in[i]==x)?1.f:0.f
template <typename DType, typename Lang>
void EQ(const Tensor &in, const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "EQ Not Implemented";
+ LOG_FATAL("EQ", DType, Lang);
}
/// out[i]=(in[i]==in2[i])?1.f:0.f
template <typename DType, typename Lang>
void EQ(const Tensor &in, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Tensor-Tensor EQ Not Implemented";
+ LOG_FATAL("Tensor Tensor EQ", DType, Lang);
}
/// out[i] = pow(in[i], x)
template <typename DType, typename Lang>
void Pow(const Tensor &in, const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Pow Not Implemented";
+ LOG_FATAL("Pow", DType, Lang);
}
/// out[i]=pow(in1[i], in2[i])
template <typename DType, typename Lang>
void Pow(const Tensor &in1, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Pow-Pair Not Implemented";
+ LOG_FATAL("Tensor Tensor Pow", DType, Lang);
}
/// out[i]=max(0, in[i])
template <typename DType, typename Lang>
void ReLU(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "ReLU Not Implemented";
+ LOG_FATAL("ReLU", DType, Lang);
}
/// out[i] = x
template <typename DType, typename Lang>
void Set(const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Set Not Implemented";
+ LOG_FATAL("Set", DType, Lang);
}
/// out[i]=sigmoid(in[i])
template <typename DType, typename Lang>
void Sigmoid(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Sigmoid Not Implemented";
+ LOG_FATAL("Sigmoid", DType, Lang);
}
/// out[i] = log(exp(in[i]) + 1)
template <typename DType, typename Lang>
void SoftPlus(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "SoftPlus Not Implemented";
+ LOG_FATAL("SoftPlus", DType, Lang);
}
/// out[i] = in[i] / (abs(in[i]) + 1)
template <typename DType, typename Lang>
void SoftSign(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "SoftSign Not Implemented";
+ LOG_FATAL("SoftSign", DType, Lang);
}
/// out[i] = sign(in[i])
template <typename DType, typename Lang>
void Sign(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Sign Not Implemented";
+ LOG_FATAL("Sign", DType, Lang);
}
/// out[i]=sqrt(in[i])
template <typename DType, typename Lang>
void Sqrt(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Sqrt Not Implemented";
+ LOG_FATAL("Sqrt", DType, Lang);
}
/// out[i]=square(in[i])
@@ -302,13 +306,13 @@
/// out[i] = in1[i] - in2[i]
template <typename DType, typename Lang>
void Sub(const Tensor &in1, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Sub-Pair Not Implemented";
+ LOG_FATAL("Tensor Tensor Sub", DType, Lang);
}
/// sum all elements of in into out
template <typename DType, typename Lang>
void Sum(const Tensor &in, DType *out, Context *ctx) {
- LOG(FATAL) << "Sum Not Implemented";
+ LOG_FATAL("Sum", DType, Lang);
}
/// out[i]=fn(in[i])
@@ -316,8 +320,7 @@
template <typename DType, typename Lang> \
void fn(const Tensor &in, Tensor *out, Context *ctx) { \
std::string str = stringfn; \
- str += " Not Implemented"; \
- LOG(FATAL) << str; \
+ LOG_FATAL(str, DType, Lang); \
}
GenUnaryNotImplemented(Cos, "Cos");
@@ -339,7 +342,7 @@
/// strides
template <typename DType, typename Lang>
void Transform(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Transform Not Implemented";
+ LOG_FATAL("Transform", DType, Lang);
}
// **************************************
@@ -350,19 +353,19 @@
// If DType is not float, then convert the threshold to DType
template <typename DType, typename Lang>
void Bernoulli(const float p, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Bernoulli Not Implemented";
+ LOG_FATAL("Bernoulli", DType, Lang);
}
// The random generator should be extracted from ctx.
// If DType is not float, then convert the mean and std to DType
template <typename DType, typename Lang>
void Gaussian(const DType mean, const DType std, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Gaussian Not Implemented";
+ LOG_FATAL("Gaussian", DType, Lang);
}
// The random generator should be extracted from ctx.
// If DType is not float, then convert the low and high to DType
template <typename DType, typename Lang>
void Uniform(const DType low, const DType high, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Uniform Not Implemented";
+ LOG_FATAL("Uniform", DType, Lang);
}
// *********************************************************
@@ -372,52 +375,52 @@
/// outurn the index of the element with the max value.
template <typename DType, typename Lang>
void Amax(const Tensor &in, size_t *out, Context *ctx) {
- LOG(FATAL) << "Amax Not Implemented";
+ LOG_FATAL("Amax", DType, Lang);
}
/// outurn the index of the element with the min value.
template <typename DType, typename Lang>
void Amin(const Tensor &in, size_t *out, Context *ctx) {
- LOG(FATAL) << "Amin Not Implemented";
+ LOG_FATAL("Amin", DType, Lang);
}
/// out = sum |x| for all x in in
template <typename DType, typename Lang>
void Asum(const Tensor &in, DType *out, Context *ctx) {
- LOG(FATAL) << "Asum Not Implemented";
+ LOG_FATAL("Asum", DType, Lang);
}
/// out = alpha * in + out
template <typename DType, typename Lang>
void Axpy(const DType alpha, const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Axpy Not Implemented";
+ LOG_FATAL("Axpy", DType, Lang);
}
/// out = alpha * in + out
template <typename DType, typename Lang>
void Axpy(const Tensor &alpha, const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Axpy Not Implemented";
+ LOG_FATAL("Axpy Tensor alpha", DType, Lang);
}
/// out = ||in||_2^2, i.e, L2 norm.
template <typename DType, typename Lang>
void Nrm2(const Tensor &in, float *out, Context *ctx) {
- LOG(FATAL) << "Nrm2 Not Implemented";
+ LOG_FATAL("Nrm2", DType, Lang);
}
/// out *= x
template <typename DType, typename Lang>
void Scale(const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Scale Not Implemented";
+ LOG_FATAL("Scale", DType, Lang);
}
/// inner product of array in1 and in2
template <typename DType, typename Lang>
void Dot(const Tensor &in1, const Tensor &in2, DType *out, Context *ctx) {
- LOG(FATAL) << "Dot Not Implemented";
+ LOG_FATAL("Inner-product Dot", DType, Lang);
}
template <typename DType, typename Lang>
void Dot(const Tensor &in1, const Tensor &in2, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Dot Not Implemented";
+ LOG_FATAL("Dot", DType, Lang);
}
/// out = alpha * A * v + beta * out.
@@ -425,7 +428,7 @@
template <typename DType, typename Lang>
void GEMV(const DType alpha, const Tensor &A, const Tensor &v, const DType beta,
Tensor *out, Context *ctx) {
- LOG(FATAL) << "GEMV Not Implemented";
+ LOG_FATAL("GEMV", DType, Lang);
}
/// multiply a matrix with a diagnoal matrix constructed using values from 'v'.
@@ -433,7 +436,7 @@
template <typename DType, typename Lang>
void DGMM(const bool side_right, const Tensor &M, const Tensor &v, Tensor *out,
Context *ctx) {
- LOG(FATAL) << "DGMM Not Implemented";
+ LOG_FATAL("DGMM", DType, Lang);
}
/// C = alpha * A * B + beta * C.
@@ -441,24 +444,24 @@
template <typename DType, typename Lang>
void GEMM(const DType alpha, const Tensor &A, const Tensor &B, const DType beta,
Tensor *C, Context *ctx) {
- LOG(FATAL) << "GEMM Not Implemented";
+ LOG_FATAL("GEMM", DType, Lang);
}
template <typename DType, typename Lang>
void GEMMBatched(const DType alpha, const Tensor &A, const Tensor &B,
const DType beta, Tensor *C, Context *ctx) {
- LOG(FATAL) << "GEMM Batched Not Implemented";
+ LOG_FATAL("GEMMBatched", DType, Lang);
}
template <typename DType, typename Lang>
void SoftMax(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Not Implemented";
+ LOG_FATAL("SoftMax", DType, Lang);
}
template <typename DType, typename Lang>
void SoftMaxBackward(const Tensor &in, Tensor *out, const Tensor &fdout,
Context *ctx) {
- LOG(FATAL) << "Not Implemented";
+ LOG_FATAL("SoftMaxBackend", DType, Lang);
}
// yisen todo
@@ -466,68 +469,20 @@
void ComputeCrossEntropy(bool int_target, const size_t batchsize,
const size_t dim, const Tensor &p, const Tensor &t,
Tensor *loss, Context *ctx) {
- LOG(FATAL) << "Not Implemented";
+ LOG_FATAL("ComputeCrossEntropy", DType, Lang);
}
template <typename DType, typename Lang>
void SoftmaxCrossEntropyBwd(bool int_target, const size_t batchsize,
const size_t dim, const Tensor &p, const Tensor &t,
Tensor *grad, Context *ctx) {
- LOG(FATAL) << "Not Implemented";
+ LOG_FATAL("ComputeCrossEntropyBwd", DType, Lang);
}
template <typename DType, typename Lang>
void RowMax(const Tensor &in, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Not Implemented";
+ LOG_FATAL("RowMax", DType, Lang);
}
-// **************************************
-// Matrix functions
-// **************************************
-/*
-/// Add the vector v to every column of A as the column of out
-template <typename DType, typename Lang>
-void AddCol(const size_t nrow, const size_t ncol, const Tensor &A, const Tensor
-&v,
- Tensor *out, Context *ctx) {
- LOG(FATAL) << "AddCol Not Implemented";
-}
-// TODO(wangwei) unify AddRow and AddCol.
-/// Add the vector v to every row of A as the row of out
-template <typename DType, typename Lang>
-void AddRow(const size_t nrow, const size_t ncol, const Tensor &A, const Tensor
-&v,
- Tensor *out, Context *ctx) {
- LOG(FATAL) << "AddRow Not Implemented";
-}
-/// outer-product.
-/// in1 and in2 are vectors of len m and n. out is matrix of shape m * n
-template <typename DType, typename Lang>
-void Outer(const size_t m, const size_t n, const Tensor &in1, const Tensor &in2,
- Tensor *out, Context *ctx) {
- LOG(FATAL) << "Outer Not Implemented";
-}
-
-/// Sum the columns of the in matrix into a vector
-template <typename DType, typename Lang>
-void SumColumns(const size_t nrow, const size_t ncol, const Tensor &in, Tensor
-*out,
- Context *ctx) {
- LOG(FATAL) << "SumColumns Not Implemented";
-}
-template <typename DType, typename Lang>
-void Set(const DType x, Tensor *out, Context *ctx) {
- LOG(FATAL) << "Not Implemented";
-}
-
-// TODO(wangwei) unify SumRow and SumCol.
-/// Sum the rows of the in matrix into a vector
-template <typename DType, typename Lang>
-void SumRows(const size_t nrow, const size_t ncol, const Tensor &in, Tensor
-*out,
- Context *ctx) {
- LOG(FATAL) << "SumRows Not Implemented";
-}
-*/
} // namespace singa
#endif // SINGA_CORE_MATH_H_
diff --git a/src/core/tensor/tensor_math_cpp.h b/src/core/tensor/tensor_math_cpp.h
index 2c06f63..b3113ab 100644
--- a/src/core/tensor/tensor_math_cpp.h
+++ b/src/core/tensor/tensor_math_cpp.h
@@ -785,20 +785,6 @@
*out = cblas_sasum(in.Size(), inPtr, 1); // not using strided traversal
}
-// template <>
-// void Axpy<float, lang::Cpp>(const float alpha,
-// const Tensor& in, Tensor *out, Context *ctx) {
-// //check input tensor for strides first
-// if (in.stride() == out->stride()) {
-// const float *inPtr = static_cast<const float *>(in.block()->data());
-// float *outPtr = static_cast<float *>(out->block()->mutable_data());
-// cblas_saxpy(in.Size(), alpha, inPtr, 1, outPtr, 1);
-// } else {
-// //LOG(FATAL) << "Axpy, input and output strides do not match." ;
-// EltwiseMult<float, lang::Cpp>(in, alpha, out, ctx);
-// }
-// }
-
template <>
void Axpy<float, lang::Cpp>(const float alpha, const Tensor &in, Tensor *out,
Context *ctx) {
@@ -817,20 +803,25 @@
}
}
-// template <>
-// void Axpy<float, lang::Cpp>(const float alpha,
-// const Tensor& in, Tensor *out, Context *ctx) {
-// //check input tensor for strides first
-// if (in.stride() == out->stride()) {
-// const float *inPtr = static_cast<const float *>(in.block()->data());
-// float *outPtr = static_cast<float *>(out->block()->mutable_data());
-// cblas_saxpy(in.Size(), alpha, inPtr, 1, outPtr, 1);
-// } else if(out->transpose()) {
-// LOG(FATAL) << "output is already transposed." ;
-// } else {
-// LOG(FATAL) << "Axpy, input and output strides do not match." ;
-// }
-// }
+template <>
+void Axpy<float, lang::Cpp>(const Tensor &alpha, const Tensor &in, Tensor *out,
+ Context *ctx) {
+ // check input tensor for strides first
+ const float *inPtr = static_cast<const float *>(in.block()->data());
+ float *outPtr = static_cast<float *>(out->block()->mutable_data());
+ const float a = *static_cast<const float*>(alpha.block()->data());
+
+ if (in.stride() == out->stride()) {
+ cblas_saxpy(in.Size(), a, inPtr, 1, outPtr, 1);
+ } else {
+ // LOG(FATAL) << "Axpy, input and output strides do not match." ;
+ Tensor t(in.shape(), in.device(), in.data_type());
+ EltwiseMult<float, lang::Cpp>(in, a, &t, ctx);
+ float *tPtr = static_cast<float *>(t.block()->mutable_data());
+ cblas_saxpy(in.Size(), 1, tPtr, 1, outPtr, 1);
+ }
+}
+
template <>
void Dot<float, lang::Cpp>(const Tensor &in1, const Tensor &in2, float *out,
@@ -1148,121 +1139,7 @@
}
}
-// =========Matrix operations ================================================
-/*
-template <>
-void SoftMax<float, lang::Cpp>(const Tensor &in, Tensor *out, Context* ctx) {
- CHECK_LE(in.nDim(), 2u) << "Axis is required for SoftMax on multi dimemsional
-tensor";
- out->CopyData(in);
- size_t nrow = 1, ncol = in.Size(), size = ncol;
- if (in.nDim() == 2u) {
- nrow = in.shape(0);
- ncol = size / nrow;
- out->Reshape(Shape{nrow, ncol});
- }
- Tensor tmp = RowMax(*out);
- SubColumn(tmp, out);
- Exp(*out, out);
- SumColumns(*out, &tmp);
- DivColumn(tmp, out);
- out->Reshape(in.shape());
-}
-
-template <>
-void AddCol<float, lang::Cpp>(const size_t nrow, const size_t ncol,
- const Tensor& A, const Tensor& v, Tensor* out,
- Context *ctx) {
- float *outPtr = static_cast<float *>(out->mutable_data());
- const float *APtr = static_cast<const float *>(A.data());
- const float *vPtr = static_cast<const float *>(v.data());
- for (size_t r = 0; r < nrow; r++) {
- size_t offset = r * ncol;
- for (size_t c = 0; c < ncol; c++) {
- outPtr[offset + c] = APtr[offset + c] + vPtr[r];
- }
- }
-}
-
-template <>
-void AddRow<float, lang::Cpp>(const size_t nrow, const size_t ncol,
- const Tensor& A, const Tensor& v, Tensor* out,
- Context *ctx) {
- float *outPtr = static_cast<float *>(out->mutable_data());
- const float *APtr = static_cast<const float *>(A.data());
- const float *vPtr = static_cast<const float *>(v.data());
- for (size_t r = 0; r < nrow; r++) {
- size_t offset = r * ncol;
- for (size_t c = 0; c < ncol; c++) {
- outPtr[offset + c] = APtr[offset + c] + vPtr[c];
- }
- }
-}
-template <>
-void Outer<float, lang::Cpp>(const size_t m, const size_t n, const Tensor& in1,
- const Tensor& in2, Tensor* out, Context *ctx) {
- float *outPtr = static_cast<float *>(out->mutable_data());
- const float *in1Ptr = static_cast<const float *>(in1.data());
- const float *in2Ptr = static_cast<const float *>(in2.data());
- for (size_t r = 0; r < m; r++) {
- size_t offset = r * n;
- for (size_t c = 0; c < n; c++) {
- outPtr[offset + c] = in1Ptr[r] * in2Ptr[c];
- }
- }
-}
-template <>
-void Softmax<float, lang::Cpp>(const size_t nrow, const size_t ncol,
- const Tensor& in, Tensor* out, Context *ctx) {
- float *outPtr = static_cast<float *>(out->mutable_data());
- const float *inPtr = static_cast<const float *>(in.data());
- float *bPtr = new float[ncol];
- for (size_t r = 0; r < nrow; r++) {
- size_t offset = r * ncol;
- float denom = 0.f;
- for (size_t c = 0; c < ncol; c++) {
- bPtr[c] = exp(inPtr[offset + c]);
- denom += bPtr[c];
- }
- for (size_t c = 0; c < ncol; c++) {
- size_t idx = offset + c;
- outPtr[idx] = bPtr[c] / denom;
- }
- }
- delete bPtr;
-}
-
-template <>
-void SumColumns<float, lang::Cpp>(const size_t nrow, const size_t ncol,
- const Tensor& in, Tensor* out, Context *ctx) {
- float *outPtr = static_cast<float *>(out->mutable_data());
- const float *inPtr = static_cast<const float *>(in.data());
- for (size_t c = 0; c < ncol; c++) {
- outPtr[c] = 0.f;
- }
- for (size_t r = 0; r < nrow; r++) {
- size_t offset = r * ncol;
- for (size_t c = 0; c < ncol; c++) {
- outPtr[c] += inPtr[offset + c];
- }
- }
-}
-
-template <>
-void SumRows<float, lang::Cpp>(const size_t nrow, const size_t ncol,
- const Tensor& in, Tensor* out, Context *ctx) {
- float *outPtr = static_cast<float *>(out->mutable_data());
- const float *inPtr = static_cast<const float *>(in.data());
- for (size_t r = 0; r < nrow; r++) {
- size_t offset = r * ncol;
- outPtr[r] = 0.f;
- for (size_t c = 0; c < ncol; c++) {
- outPtr[r] += inPtr[offset + c];
- }
- }
-}
-*/
} // namespace singa
#endif // SINGA_CORE_TENSOR_TENSOR_MATH_CPP_H_
diff --git a/test/python/test_tensor.py b/test/python/test_tensor.py
index 82d6d5c..f7044f1 100644
--- a/test/python/test_tensor.py
+++ b/test/python/test_tensor.py
@@ -19,6 +19,7 @@
import math
import unittest
+import random
import numpy as np
from singa import tensor
@@ -546,10 +547,10 @@
x_val = np.random.randint(0, 10, (2, 3))
x = tensor.from_numpy(x_val)
x.to_device(dev)
- scalar = np.random.random((1,))[0] * 100
+ scalar = random.random() * 100
y = x + scalar
self.assertEqual(y.dtype, tensor.float32)
- np.testing.assert_array_almost_equal(tensor.to_numpy(y), x_val + scalar)
+ np.testing.assert_array_almost_equal(tensor.to_numpy(y), x_val + scalar, 5)
@unittest.skipIf(not singa_api.USE_CUDA, 'CUDA is not enabled')
def test_kint_float_gpu(self):
diff --git a/tool/conda/gpu/meta.yaml b/tool/conda/gpu/meta.yaml
index 58ef499..68df6b3 100644
--- a/tool/conda/gpu/meta.yaml
+++ b/tool/conda/gpu/meta.yaml
@@ -26,7 +26,7 @@
requirements:
run:
- - singa {{ environ.get('GIT_DESCRIBE_TAG') | replace("-", ".") }} cudnn7.6.5_cuda10.0_py{{ py }}
+ - singa {{ environ.get('GIT_DESCRIBE_TAG') | replace("-", ".") }} cudnn7.6.5_cuda10.2_nccl2.6.4.1_mpich3.3.2_py{{ py }}
build:
number: 0
diff --git a/tool/conda/singa/meta.yaml b/tool/conda/singa/meta.yaml
index cface0d..5cacf3b 100644
--- a/tool/conda/singa/meta.yaml
+++ b/tool/conda/singa/meta.yaml
@@ -20,7 +20,7 @@
# https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#templating-with-jinja
# {% set data = load_setup_py_data(setup_file='../../../python/singa/setup.py', from_recipe_dir=True) %}
-{% set version = "2.1.0.dev" %}
+{% set version = "3.2.0" %}
package:
name: singa
diff --git a/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux2014 b/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux2014
index d3aeaff..80ca788 100644
--- a/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux2014
+++ b/tool/docker/devel/centos6/cuda10/Dockerfile.manylinux2014
@@ -119,6 +119,14 @@
rm cudnn-10.2-linux-x64-v7.6.5.32.tgz && \
ldconfig
+# install nccl for distributed training
+RUN git clone https://github.com/NVIDIA/nccl.git $HOME/nccl \
+ && cd $HOME/nccl \
+ && git checkout v2.4.8-1 \
+ && make BUILDDIR=/usr/local/ -j$(nproc) src.build \
+ && rm -rf /usr/local/obj \
+ && rm -rf $HOME/nccl
+
# install cnmem to /usr/local/include /usr/local/lib
RUN git clone https://github.com/NVIDIA/cnmem.git cnmem \
&& cd cnmem && mkdir build && cd build && cmake .. && make && make install && cd ../.. && rm -rf cnmem
