src/target/intrin_rule.cc

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*!
 * \file intrin_rule_default.cc
 * \brief Default intrinsic rules.
 */
#include "intrin_rule.h"

#include <tvm/runtime/logging.h>
#include <tvm/tirx/buffer.h>
#include <tvm/tirx/op.h>
#include <tvm/tirx/op_attr_types.h>

namespace tvm {
namespace codegen {
namespace intrin {
using tirx::FLowerIntrinsic;

TVM_REGISTER_OP("tirx.exp")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.erf")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.log")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.log2")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.log10")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.log1p")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.tanh")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.tan")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.trunc")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.atan")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.atanh")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.atan2")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.cos")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.acos")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.cosh")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.acosh")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.sin")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.asin")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.sinh")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.asinh")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.hypot")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.nextafter")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.copysign")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.ldexp")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.sqrt")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.floor")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.ceil")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.round")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.nearbyint")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

TVM_REGISTER_OP("tirx.pow")
    .set_attr<FLowerIntrinsic>("default.FLowerIntrinsic", DispatchPureExtern<FloatSuffix>);

PrimExpr DispatchFastErf(const PrimExpr& e) {
  DLOG(WARNING) << "fast_erf will be used instead of erf";
  const CallNode* call = e.as<CallNode>();
  TVM_FFI_ICHECK(call != nullptr);
  TVM_FFI_ICHECK_EQ(call->args.size(), 1);
  PrimExpr arg = call->args[0].as_or_throw<PrimExpr>();
  PrimType arg_ty = arg.ty();
  int bits = arg_ty.bits();
  PrimExpr res;
  if (arg_ty.code() == DLDataTypeCode::kDLFloat && (bits == 16 || bits == 32)) {
    res = fast_erf_float_expr(arg, bits);
  } else {
    TVM_FFI_THROW(InternalError) << "Unsupported type in Metal fast_erf";
  }
  return res;
}

PrimExpr DispatchNumericalStableTanh(const PrimExpr& e) {
  using tirx::MakeConst;
  const CallNode* call = e.as<CallNode>();
  TVM_FFI_ICHECK(call != nullptr);
  PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
  PrimType x_ty = x.ty();
  PrimExpr one = MakeConst(x_ty, 1);
  PrimExpr two = MakeConst(x_ty, 2);
  PrimExpr neg_two = MakeConst(x_ty, -2);

  PrimExpr exp_neg2x = exp(neg_two * x);
  PrimExpr exp_pos2x = exp(two * x);

  PrimExpr tanh_pos = (one - exp_neg2x) / (one + exp_neg2x);
  PrimExpr tanh_neg = (exp_pos2x - one) / (exp_pos2x + one);
  // MakeConst can handle both vector and scalar types.
  return tirx::Select(x >= MakeConst(x_ty, 0), tanh_pos, tanh_neg);
}

}  // namespace intrin

namespace legalize {

using namespace tirx;

TVM_REGISTER_OP("tirx.rsqrt")
    .set_attr<FLegalize>("default.FLegalize", [](const PrimExpr& e) -> PrimExpr {
      const CallNode* call = e.as<CallNode>();
      TVM_FFI_ICHECK(call != nullptr);
      PrimExpr arg = call->args[0].as_or_throw<PrimExpr>();
      auto one = MakeConst(arg.ty(), 1);
      return one / sqrt(arg);
    });

TVM_REGISTER_OP("tirx.sigmoid")
    .set_attr<FLegalize>("default.FLegalize", [](const PrimExpr& e) -> PrimExpr {
      const CallNode* call = e.as<CallNode>();
      TVM_FFI_ICHECK(call != nullptr);
      PrimExpr arg = call->args[0].as_or_throw<PrimExpr>();
      auto one = MakeConst(arg.ty(), 1);
      return one / (one + exp(-arg));
    });

TVM_REGISTER_OP("tirx.isfinite")
    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), 1)
    .set_attr<FLegalize>("default.FLegalize", [](const PrimExpr& e) -> PrimExpr {
      const CallNode* call = e.as<CallNode>();
      TVM_FFI_ICHECK(call != nullptr);
      return isfinite(call->args[0].as_or_throw<PrimExpr>());
    });

TVM_REGISTER_OP("tirx.isinf")
    .set_attr<tirx::TIRxOpCategory>("TIRxOpCategory", ffi::String("builtin"), 1)
    .set_attr<FLegalize>("default.FLegalize", [](const PrimExpr& e) -> PrimExpr {
      const CallNode* call = e.as<CallNode>();
      TVM_FFI_ICHECK(call != nullptr);
      return isinf(call->args[0].as_or_throw<PrimExpr>());
    });

/*!
 * \brief Makes fixed point multiplication.
 * \param x Input tensor.
 * \param y Integer multiplier.
 * \param left_shift Integer left shift.
 * \param right_shift Integer right shift.
 * \param is_left_shift_required Flag whether we need to do left shift or not.
 * \return Calculated expression.
 */
static PrimExpr QMultiplyShift(PrimExpr x, PrimExpr y, PrimExpr q, PrimExpr left_shift,
                               PrimExpr right_shift, PrimExpr is_left_shift_required) {
  // Only int32 types are supported (any number of lanes is allowed)
  TVM_FFI_ICHECK(y.ty().MatchesElementType(DLDataTypeCode::kDLInt, 32));
  TVM_FFI_ICHECK(left_shift.ty().MatchesElementType(DLDataTypeCode::kDLInt, 32));
  TVM_FFI_ICHECK(right_shift.ty().MatchesElementType(DLDataTypeCode::kDLInt, 32));

  PrimType x_ty = x.ty();
  auto signed_int_ty = [](int bits, const PrimType& source_ty) {
    if (source_ty.IsScalableVector()) {
      return PrimType::ScalableVector(DLDataTypeCode::kDLInt, bits, source_ty.VScaleFactor());
    }
    return PrimType::Int(bits, source_ty.lanes());
  };
  PrimType hp_dtype = signed_int_ty(64, x_ty);
  PrimType lp_dtype = signed_int_ty(32, x_ty);

  // 1) Cast and Multiply the integer multiplier
  PrimExpr one = MakeConst(hp_dtype, 1);
  x = cast(hp_dtype, x);
  y = cast(hp_dtype, y);
  x = tirx::Select(is_left_shift_required, x << left_shift, x);

  // 2) Perform the multiplication in higher precision.
  x = x * y;

  // 3) Find the rounding scalar
  PrimExpr total_right_shift = right_shift + q;
  PrimExpr pos_rounding_value = (one << (total_right_shift - 1));
  x = x + pos_rounding_value;

  // 4) Simply right shift the result to get the final output.
  x = x >> total_right_shift;

  // 5) The fixed point multiplication keeps the value in int32 range. Casting back to int32.
  return cast(lp_dtype, x);
}

TVM_REGISTER_OP("tirx.q_multiply_shift")
    .set_attr<FLegalize>("default.FLegalize", [](const PrimExpr& e) -> PrimExpr {
      using tirx::MakeConst;

      const CallNode* call = e.as<CallNode>();
      TVM_FFI_ICHECK(call != nullptr);

      PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
      PrimExpr y = call->args[1].as_or_throw<PrimExpr>();
      PrimExpr q = call->args[2].as_or_throw<PrimExpr>();
      PrimExpr s = call->args[3].as_or_throw<PrimExpr>();

      // Lambda function to extract the int value from PrimExpr
      auto get_int_value = [](const PrimExpr node) {
        if (auto int_node = node.as<IntImmNode>()) {
          return int_node->value;
        }
        auto broadcast_node = node.as<BroadcastNode>();
        TVM_FFI_ICHECK(broadcast_node != nullptr);
        auto int_node = broadcast_node->value.as<IntImmNode>();
        TVM_FFI_ICHECK(int_node != nullptr);
        return int_node->value;
      };
      // Power of 2 is determined by the fixed_point_multiplier == 1 << 30. In case of power of
      // 2, fixed point multiplier will represent a float value of 0.5. In fixed point, this is
      // represented by 1 << 30.
      if (get_int_value(y) == (1 << 30)) {
        PrimExpr exp = s - 1;
        int exp_val = get_int_value(s) - 1;
        if (exp_val > 0) {
          // power of 2 is greater than 0, apply left shift.
          return x << exp;
        } else {
          // power of 2 is less than 0, round and then apply right shift.
          PrimType x_ty = x.ty();
          PrimType lp_dtype =
              x_ty.IsScalableVector()
                  ? PrimType::ScalableVector(DLDataTypeCode::kDLInt, 32, x_ty.VScaleFactor())
                  : PrimType::Int(32, x_ty.lanes());
          PrimExpr one = MakeConst(lp_dtype, 1);
          exp = -exp;
          PrimExpr rounding_factor = one << (exp - 1);
          PrimExpr rounded_t = x + rounding_factor;
          return rounded_t >> exp;
        }
      } else {
        // Only int32 types are supported (any number of lanes is allowed)
        TVM_FFI_ICHECK(s.ty().MatchesElementType(DLDataTypeCode::kDLInt, 32));

        // Calculating integer shifts. MakeConst can handle both vector and scalar types.
        PrimType s_ty = s.ty();
        PrimExpr zero = MakeConst(s_ty, 0);
        PrimExpr left_shift = tirx::Select(s > zero, s, zero);
        PrimExpr right_shift = tirx::Select(s > zero, zero, -s);
        PrimExpr is_left_shift_required = (left_shift != zero);

        return QMultiplyShift(x, y, q, left_shift, right_shift, is_left_shift_required);
      }
    });

TVM_REGISTER_OP("tirx.q_multiply_shift_per_axis")
    .set_attr<FLegalize>("default.FLegalize", [](const PrimExpr& e) -> PrimExpr {
      const CallNode* call = e.as<CallNode>();
      TVM_FFI_ICHECK(call != nullptr);

      PrimExpr x = call->args[0].as_or_throw<PrimExpr>();
      PrimExpr y = call->args[1].as_or_throw<PrimExpr>();
      PrimExpr left_shift = call->args[2].as_or_throw<PrimExpr>();
      PrimExpr right_shift = call->args[3].as_or_throw<PrimExpr>();
      PrimExpr q = call->args[4].as_or_throw<PrimExpr>();
      PrimExpr is_lshift_required = call->args[5].as_or_throw<PrimExpr>();
      // Note, 7th argument is "is_rshift_required" flag, but we don't need that here.
      // PrimExpr is_rshift_required = call->args[6];

      return QMultiplyShift(x, y, q, left_shift, right_shift, is_lshift_required);
    });
}  // namespace legalize
}  // namespace codegen
}  // namespace tvm