be/src/runtime/tuple.cc

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

#include "runtime/tuple.h"

#include <vector>
#include "llvm/IR/Function.h"

#include "codegen/codegen-anyval.h"
#include "codegen/llvm-codegen.h"
#include "exprs/scalar-expr-evaluator.h"
#include "exprs/scalar-expr.h"
#include "gutil/strings/substitute.h"
#include "runtime/collection-value.h"
#include "runtime/descriptors.h"
#include "runtime/mem-pool.h"
#include "runtime/mem-tracker.h"
#include "runtime/raw-value.h"
#include "runtime/runtime-state.h"
#include "runtime/string-value.h"
#include "runtime/tuple-row.h"
#include "util/debug-util.h"
#include "util/runtime-profile-counters.h"
#include "util/ubsan.h"

#include "common/names.h"

namespace impala {

const char* Tuple::LLVM_CLASS_NAME = "class.impala::Tuple";
const char* SlotOffsets::LLVM_CLASS_NAME = "struct.impala::SlotOffsets";

const char* Tuple::MATERIALIZE_EXPRS_SYMBOL = "MaterializeExprsILb0ELb0";
const char* Tuple::MATERIALIZE_EXPRS_NULL_POOL_SYMBOL = "MaterializeExprsILb0ELb1";

Tuple* const Tuple::POISON = reinterpret_cast<Tuple*>(42L);

int64_t Tuple::TotalByteSize(const TupleDescriptor& desc) const {
  int64_t result = desc.byte_size();
  if (!desc.HasVarlenSlots()) return result;
  result += VarlenByteSize(desc);
  return result;
}

int64_t Tuple::VarlenByteSize(const TupleDescriptor& desc) const {
  int64_t result = 0;
  vector<SlotDescriptor*>::const_iterator slot = desc.string_slots().begin();
  for (; slot != desc.string_slots().end(); ++slot) {
    DCHECK((*slot)->type().IsVarLenStringType());
    if (IsNull((*slot)->null_indicator_offset())) continue;
    const StringValue* string_val = GetStringSlot((*slot)->tuple_offset());
    result += string_val->len;
  }

  slot = desc.collection_slots().begin();
  for (; slot != desc.collection_slots().end(); ++slot) {
    DCHECK((*slot)->type().IsCollectionType());
    if (IsNull((*slot)->null_indicator_offset())) continue;
    const CollectionValue* coll_value = GetCollectionSlot((*slot)->tuple_offset());
    uint8_t* coll_data = coll_value->ptr;
    const TupleDescriptor& item_desc = *(*slot)->collection_item_descriptor();
    for (int i = 0; i < coll_value->num_tuples; ++i) {
      result += reinterpret_cast<Tuple*>(coll_data)->TotalByteSize(item_desc);
      coll_data += item_desc.byte_size();
    }
  }
  return result;
}

Tuple* Tuple::DeepCopy(const TupleDescriptor& desc, MemPool* pool) {
  Tuple* result = reinterpret_cast<Tuple*>(pool->Allocate(desc.byte_size()));
  DeepCopy(result, desc, pool);
  return result;
}

// TODO: the logic is very similar to the other DeepCopy implementation aside from how
// memory is allocated - can we templatise it somehow to avoid redundancy without runtime
// overhead.
void Tuple::DeepCopy(Tuple* dst, const TupleDescriptor& desc, MemPool* pool) {
  memcpy(dst, this, desc.byte_size());
  if (desc.HasVarlenSlots()) dst->DeepCopyVarlenData(desc, pool);
}

void Tuple::DeepCopyVarlenData(const TupleDescriptor& desc, MemPool* pool) {
  // allocate then copy all non-null string and collection slots
  for (vector<SlotDescriptor*>::const_iterator slot = desc.string_slots().begin();
       slot != desc.string_slots().end(); ++slot) {
    DCHECK((*slot)->type().IsVarLenStringType());
    if (IsNull((*slot)->null_indicator_offset())) continue;
    StringValue* string_v = GetStringSlot((*slot)->tuple_offset());
    char* string_copy = reinterpret_cast<char*>(pool->Allocate(string_v->len));
    Ubsan::MemCpy(string_copy, string_v->ptr, string_v->len);
    string_v->ptr = string_copy;
  }

  for (vector<SlotDescriptor*>::const_iterator slot = desc.collection_slots().begin();
       slot != desc.collection_slots().end(); ++slot) {
    DCHECK((*slot)->type().IsCollectionType());
    if (IsNull((*slot)->null_indicator_offset())) continue;
    CollectionValue* cv = GetCollectionSlot((*slot)->tuple_offset());
    const TupleDescriptor* item_desc = (*slot)->collection_item_descriptor();
    int coll_byte_size = cv->num_tuples * item_desc->byte_size();
    uint8_t* coll_data = reinterpret_cast<uint8_t*>(pool->Allocate(coll_byte_size));
    memcpy(coll_data, cv->ptr, coll_byte_size);
    cv->ptr = coll_data;
    if (!item_desc->HasVarlenSlots()) continue;

    for (int i = 0; i < cv->num_tuples; ++i) {
      int item_offset = i * item_desc->byte_size();
      Tuple* dst_item = reinterpret_cast<Tuple*>(coll_data + item_offset);
      dst_item->DeepCopyVarlenData(*item_desc, pool);
    }
  }
}

void Tuple::DeepCopy(const TupleDescriptor& desc, char** data, int* offset,
                     bool convert_ptrs) {
  Tuple* dst = reinterpret_cast<Tuple*>(*data);
  memcpy(dst, this, desc.byte_size());
  *data += desc.byte_size();
  *offset += desc.byte_size();
  if (desc.HasVarlenSlots()) dst->DeepCopyVarlenData(desc, data, offset, convert_ptrs);
}

void Tuple::DeepCopyVarlenData(const TupleDescriptor& desc, char** data, int* offset,
    bool convert_ptrs) {
  vector<SlotDescriptor*>::const_iterator slot = desc.string_slots().begin();
  for (; slot != desc.string_slots().end(); ++slot) {
    DCHECK((*slot)->type().IsVarLenStringType());
    if (IsNull((*slot)->null_indicator_offset())) continue;

    StringValue* string_v = GetStringSlot((*slot)->tuple_offset());
    Ubsan::MemCpy(*data, string_v->ptr, string_v->len);
    string_v->ptr = convert_ptrs ? reinterpret_cast<char*>(*offset) : *data;
    *data += string_v->len;
    *offset += string_v->len;
  }

  slot = desc.collection_slots().begin();
  for (; slot != desc.collection_slots().end(); ++slot) {
    DCHECK((*slot)->type().IsCollectionType());
    if (IsNull((*slot)->null_indicator_offset())) continue;

    CollectionValue* coll_value = GetCollectionSlot((*slot)->tuple_offset());
    const TupleDescriptor& item_desc = *(*slot)->collection_item_descriptor();
    int coll_byte_size = coll_value->num_tuples * item_desc.byte_size();
    memcpy(*data, coll_value->ptr, coll_byte_size);
    uint8_t* coll_data = reinterpret_cast<uint8_t*>(*data);

    coll_value->ptr = convert_ptrs ? reinterpret_cast<uint8_t*>(*offset) : coll_data;

    *data += coll_byte_size;
    *offset += coll_byte_size;

    // Copy per-tuple varlen data if necessary.
    if (!item_desc.HasVarlenSlots()) continue;
    for (int i = 0; i < coll_value->num_tuples; ++i) {
      reinterpret_cast<Tuple*>(coll_data)->DeepCopyVarlenData(
          item_desc, data, offset, convert_ptrs);
      coll_data += item_desc.byte_size();
    }
  }
}

void Tuple::ConvertOffsetsToPointers(const TupleDescriptor& desc, uint8_t* tuple_data) {
  vector<SlotDescriptor*>::const_iterator slot = desc.string_slots().begin();
  for (; slot != desc.string_slots().end(); ++slot) {
    DCHECK((*slot)->type().IsVarLenStringType());
    if (IsNull((*slot)->null_indicator_offset())) continue;

    StringValue* string_val = GetStringSlot((*slot)->tuple_offset());
    int offset = reinterpret_cast<intptr_t>(string_val->ptr);
    string_val->ptr = reinterpret_cast<char*>(tuple_data + offset);
  }

  slot = desc.collection_slots().begin();
  for (; slot != desc.collection_slots().end(); ++slot) {
    DCHECK((*slot)->type().IsCollectionType());
    if (IsNull((*slot)->null_indicator_offset())) continue;

    CollectionValue* coll_value = GetCollectionSlot((*slot)->tuple_offset());
    int offset = reinterpret_cast<intptr_t>(coll_value->ptr);
    coll_value->ptr = tuple_data + offset;

    uint8_t* coll_data = coll_value->ptr;
    const TupleDescriptor& item_desc = *(*slot)->collection_item_descriptor();
    for (int i = 0; i < coll_value->num_tuples; ++i) {
      reinterpret_cast<Tuple*>(coll_data)->ConvertOffsetsToPointers(
          item_desc, tuple_data);
      coll_data += item_desc.byte_size();
    }
  }
}

void Tuple::SetNullIndicators(NullIndicatorOffset offset, int64_t num_tuples,
    int64_t tuple_stride, uint8_t* tuple_mem) {
  // 'offset' is passed by value instead of const reference so that the compiler knows
  // that it doesn't alias 'tuple_mem' and keep the offset values in registers in the
  // below loop.
  for (int64_t i = 0; i < num_tuples; ++i) {
    reinterpret_cast<Tuple*>(tuple_mem)->SetNull(offset);
    tuple_mem += tuple_stride;
  }
}

template <bool COLLECT_STRING_VALS, bool NO_POOL>
void Tuple::MaterializeExprs(TupleRow* row, const TupleDescriptor& desc,
    ScalarExprEvaluator* const* evals, MemPool* pool,
    StringValue** non_null_string_values, int* total_string_lengths,
    int* num_non_null_string_values) {
  ClearNullBits(desc);
  // Evaluate the materialize_expr_evals and place the results in the tuple.
  for (int i = 0; i < desc.slots().size(); ++i) {
    SlotDescriptor* slot_desc = desc.slots()[i];
    // The FE ensures we don't get any TYPE_NULL expressions by picking an arbitrary type
    // when necessary, but does not do this for slot descs.
    // TODO: revisit this logic in the FE
    DCHECK(slot_desc->type().type == TYPE_NULL ||
        slot_desc->type() == evals[i]->root().type());
    void* src = evals[i]->GetValue(row);
    if (src != NULL) {
      void* dst = GetSlot(slot_desc->tuple_offset());
      RawValue::Write(src, dst, slot_desc->type(), pool);
      if (COLLECT_STRING_VALS && slot_desc->type().IsVarLenStringType()) {
        StringValue* string_val = reinterpret_cast<StringValue*>(dst);
        *(non_null_string_values++) = string_val;
        *total_string_lengths += string_val->len;
        ++(*num_non_null_string_values);
      }
    } else {
      SetNull(slot_desc->null_indicator_offset());
    }
  }
}

char* Tuple::AllocateStrings(const char* err_ctx, RuntimeState* state,
    int64_t bytes, MemPool* pool, Status* status) noexcept {
  char* buf = reinterpret_cast<char*>(pool->TryAllocateUnaligned(bytes));
  if (UNLIKELY(buf == nullptr)) {
    string details = Substitute("$0 failed to allocate $1 bytes for strings.",
        err_ctx, bytes);
    *status = pool->mem_tracker()->MemLimitExceeded(state, details, bytes);
    return nullptr;
  }
  return buf;
}

// Codegens an unrolled version of MaterializeExprs(). Uses codegen'd exprs and slot
// writes. If 'pool' is non-NULL, string data is copied into it.
//
// Example IR for materializing a string column with non-NULL 'pool':
//
// ; Function Attrs: alwaysinline
// define void @MaterializeExprs(%"class.impala::Tuple"* %opaque_tuple,
//     %"class.impala::TupleRow"* %row, %"class.impala::TupleDescriptor"* %desc,
//     %"class.impala::ScalarExprEvaluator"** %materialize_expr_evals,
//     %"class.impala::MemPool"* %pool,
//     %"struct.impala::StringValue"** %non_null_string_values,
//     i32* %total_string_lengths, i32* %num_non_null_string_values) #34 {
// entry:
//   %tuple = bitcast %"class.impala::Tuple"* %opaque_tuple to
//       <{ %"struct.impala::StringValue", i8 }>*
//   %int8_ptr = bitcast <{ %"struct.impala::StringValue", i8 }>* %tuple to i8*
//   %null_bytes_ptr = getelementptr inbounds i8, i8* %int8_ptr, i32 16
//   call void @llvm.memset.p0i8.i64(i8* %null_bytes_ptr, i8 0, i64 1, i32 0, i1 false)
//   %0 = getelementptr %"class.impala::ExprContext"*,
//       %"class.impala::ExprContext"** %materialize_expr_ctxs, i32 0
//   %expr_ctx = load %"class.impala::ExprContext"*, %"class.impala::ExprContext"** %0
//   %src = call { i64, i8* } @"impala::StringFunctions::UpperWrapper"(
//        %"class.impala::ExprContext"* %expr_ctx, %"class.impala::TupleRow"* %row)
//   %1 = extractvalue { i64, i8* } %src, 0
//   ; ----- generated by CodegenAnyVal::WriteToSlot() ----------------------------------
//   %is_null = trunc i64 %1 to i1
//   br i1 %is_null, label %null, label %non_null
//
// non_null:                                         ; preds = %entry
//   %slot = getelementptr inbounds <{ %"struct.impala::StringValue", i8 }>,
//       <{ %"struct.impala::StringValue", i8 }>* %tuple, i32 0, i32 0
//   %2 = extractvalue { i64, i8* } %src, 0
//   %3 = ashr i64 %2, 32
//   %4 = trunc i64 %3 to i32
//   %5 = insertvalue %"struct.impala::StringValue" zeroinitializer, i32 %4, 1
//   %6 = sext i32 %4 to i64
//   %new_ptr = call i8* @_ZN6impala7MemPool8AllocateILb0EEEPhli(
//       %"class.impala::MemPool"* %pool, i64 %6, i32 8)
//   %src1 = extractvalue { i64, i8* } %src, 1
//   call void @llvm.memcpy.p0i8.p0i8.i32(
//       i8* %new_ptr, i8* %src1, i32 %4, i32 0, i1 false)
//   %7 = insertvalue %"struct.impala::StringValue" %5, i8* %new_ptr, 0
//   store %"struct.impala::StringValue" %7, %"struct.impala::StringValue"* %slot
//   br label %end_write
//
// null:                                             ; preds = %entry
//   %8 = bitcast <{ %"struct.impala::StringValue", i8 }>* %tuple to i8*
//   %null_byte_ptr = getelementptr inbounds i8, i8* %8, i32 16
//   %null_byte = load i8, i8* %null_byte_ptr
//   %null_bit_set = or i8 %null_byte, 1
//   store i8 %null_bit_set, i8* %null_byte_ptr
//   br label %end_write
//
// end_write:                                        ; preds = %null, %non_null
//   ; ----- end CodegenAnyVal::WriteToSlot() -------------------------------------------
//   ret void
// }
Status Tuple::CodegenMaterializeExprs(LlvmCodeGen* codegen, bool collect_string_vals,
    const TupleDescriptor& desc, const vector<ScalarExpr*>& slot_materialize_exprs,
    bool use_mem_pool, llvm::Function** fn) {
  // Only support 'collect_string_vals' == false for now.
  if (collect_string_vals) {
    return Status("CodegenMaterializeExprs() collect_string_vals == true NYI");
  }
  llvm::LLVMContext& context = codegen->context();

  // Codegen each compute function from slot_materialize_exprs
  llvm::Function* materialize_expr_fns[slot_materialize_exprs.size()];
  for (int i = 0; i < slot_materialize_exprs.size(); ++i) {
    Status status = slot_materialize_exprs[i]->GetCodegendComputeFn(
        codegen, false, &materialize_expr_fns[i]);
    if (!status.ok()) {
      return Status::Expected(Substitute("Could not codegen CodegenMaterializeExprs: $0",
            status.GetDetail()));
    }
  }

  // Construct function signature (this must exactly match the actual signature since it's
  // used in xcompiled IR). With 'pool':
  // void MaterializeExprs(Tuple* tuple, TupleRow* row, TupleDescriptor* desc,
  //     ScalarExprEvaluator** slot_materialize_exprs, MemPool* pool,
  //     StringValue** non_null_string_values, int* total_string_lengths)
  llvm::PointerType* opaque_tuple_type = codegen->GetStructPtrType<Tuple>();
  llvm::PointerType* row_type = codegen->GetStructPtrType<TupleRow>();
  llvm::PointerType* desc_type = codegen->GetStructPtrType<TupleDescriptor>();
  llvm::PointerType* expr_evals_type =
      codegen->GetStructPtrPtrType<ScalarExprEvaluator>();
  llvm::PointerType* pool_type = codegen->GetStructPtrType<MemPool>();
  llvm::PointerType* string_values_type = codegen->GetStructPtrPtrType<StringValue>();
  llvm::PointerType* int_ptr_type = codegen->i32_ptr_type();
  LlvmCodeGen::FnPrototype prototype(codegen, "MaterializeExprs", codegen->void_type());
  prototype.AddArgument("opaque_tuple", opaque_tuple_type);
  prototype.AddArgument("row", row_type);
  prototype.AddArgument("desc", desc_type);
  prototype.AddArgument("slot_materialize_exprs", expr_evals_type);
  prototype.AddArgument("pool", pool_type);
  prototype.AddArgument("non_null_string_values", string_values_type);
  prototype.AddArgument("total_string_lengths", int_ptr_type);
  prototype.AddArgument("num_non_null_string_values", int_ptr_type);

  LlvmBuilder builder(context);
  llvm::Value* args[8];
  *fn = prototype.GeneratePrototype(&builder, args);
  llvm::Value* opaque_tuple_arg = args[0];
  llvm::Value* row_arg = args[1];
  // llvm::Value* desc_arg = args[2]; // unused
  llvm::Value* expr_evals_arg = args[3];
  llvm::Value* pool_arg = args[4];
  // The followings arguments are unused as 'collect_string_vals' is false.
  // llvm::Value* non_null_string_values_arg = args[5]; // unused
  // llvm::Value* total_string_lengths_arg = args[6]; // unused
  // llvm::Value* num_non_null_string_values_arg = args[7]; // unused

  // Cast the opaque Tuple* argument to the generated struct type
  llvm::Type* tuple_struct_type = desc.GetLlvmStruct(codegen);
  if (tuple_struct_type == NULL) {
    return Status("CodegenMaterializeExprs(): failed to generate tuple desc");
  }
  llvm::PointerType* tuple_type = codegen->GetPtrType(tuple_struct_type);
  llvm::Value* tuple = builder.CreateBitCast(opaque_tuple_arg, tuple_type, "tuple");

  // Clear tuple's null bytes
  codegen->CodegenClearNullBits(&builder, tuple, desc);

  // Evaluate the slot_materialize_exprs and place the results in the tuple.
  for (int i = 0; i < desc.slots().size(); ++i) {
    SlotDescriptor* slot_desc = desc.slots()[i];
    DCHECK(slot_desc->type().type == TYPE_NULL
        || slot_desc->type() == slot_materialize_exprs[i]->type());

    // Call materialize_expr_fns[i](slot_materialize_exprs[i], row)
    llvm::Value* expr_eval =
        codegen->CodegenArrayAt(&builder, expr_evals_arg, i, "expr_eval");
    llvm::Value* expr_args[] = {expr_eval, row_arg};
    CodegenAnyVal src = CodegenAnyVal::CreateCallWrapped(codegen, &builder,
        slot_materialize_exprs[i]->type(), materialize_expr_fns[i], expr_args, "src");

    // Write expr result 'src' to slot
    src.WriteToSlot(*slot_desc, tuple, use_mem_pool ? pool_arg : nullptr);
  }
  builder.CreateRetVoid();
  // TODO: if pool != NULL, OptimizeFunctionWithExprs() is inlining the Allocate()
  // call. Investigate if this is a good thing.
  *fn = codegen->FinalizeFunction(*fn);
  if (*fn == nullptr) {
    return Status("Tuple::CodegenMaterializeTuple(): failed to finalize function");
  }
  return Status::OK();
}

Status Tuple::CodegenCopyStrings(
    LlvmCodeGen* codegen, const TupleDescriptor& desc, llvm::Function** copy_strings_fn) {
  llvm::PointerType* opaque_tuple_type = codegen->GetStructPtrType<Tuple>();
  llvm::PointerType* runtime_state_type = codegen->GetStructPtrType<RuntimeState>();
  llvm::StructType* slot_offsets_type = codegen->GetStructType<SlotOffsets>();
  llvm::PointerType* pool_type = codegen->GetStructPtrType<MemPool>();
  llvm::PointerType* status_type = codegen->GetStructPtrType<Status>();
  LlvmCodeGen::FnPrototype prototype(
      codegen, "CopyStringsWrapper", codegen->bool_type());
  prototype.AddArgument("opaque_tuple", opaque_tuple_type);
  prototype.AddArgument("err_ctx", codegen->ptr_type());
  prototype.AddArgument("state", runtime_state_type);
  prototype.AddArgument("slot_offsets", codegen->GetPtrType(slot_offsets_type));
  prototype.AddArgument("num_string_slots", codegen->i32_type());
  prototype.AddArgument("pool", pool_type);
  prototype.AddArgument("status", status_type);

  LlvmBuilder builder(codegen->context());
  llvm::Value* args[7];
  *copy_strings_fn = prototype.GeneratePrototype(&builder, args);
  llvm::Value* opaque_tuple_arg = args[0];
  llvm::Value* err_ctx_arg = args[1];
  llvm::Value* state_arg = args[2];
  // slot_offsets and num_string_slots are replaced with constants so args are unused.
  llvm::Value* pool_arg = args[5];
  llvm::Value* status_arg = args[6];

  llvm::Function* cross_compiled_fn =
      codegen->GetFunction(IRFunction::TUPLE_COPY_STRINGS, false);
  DCHECK(cross_compiled_fn != nullptr);

  // Convert the offsets of the string slots into a constant IR array 'slot_offsets'.
  vector<llvm::Constant*> slot_offset_ir_constants;
  for (SlotDescriptor* slot_desc : desc.string_slots()) {
    SlotOffsets offsets = {slot_desc->null_indicator_offset(), slot_desc->tuple_offset()};
    slot_offset_ir_constants.push_back(offsets.ToIR(codegen));
  }
  llvm::Constant* constant_slot_offsets = codegen->ConstantsToGVArrayPtr(
      slot_offsets_type, slot_offset_ir_constants, "slot_offsets");
  llvm::Constant* num_string_slots = codegen->GetI32Constant(desc.string_slots().size());
  // Get SlotOffsets* pointer to the first element of the constant array.
  llvm::Value* constant_slot_offsets_first_element_ptr =
      builder.CreateConstGEP2_64(constant_slot_offsets, 0, 0);

  llvm::Value* result_val = builder.CreateCall(cross_compiled_fn,
      {opaque_tuple_arg, err_ctx_arg, state_arg, constant_slot_offsets_first_element_ptr,
          num_string_slots, pool_arg, status_arg});
  builder.CreateRet(result_val);

  *copy_strings_fn = codegen->FinalizeFunction(*copy_strings_fn);
  if (*copy_strings_fn == nullptr) {
    return Status("Tuple::CodegenCopyStrings(): failed to finalize function");
  }
  return Status::OK();
}

llvm::Constant* SlotOffsets::ToIR(LlvmCodeGen* codegen) const {
  return llvm::ConstantStruct::get(
      codegen->GetStructType<SlotOffsets>(),
      {null_indicator_offset.ToIR(codegen),
          codegen->GetI32Constant(tuple_offset)});
}

template void Tuple::MaterializeExprs<false, false>(TupleRow*, const TupleDescriptor&,
    ScalarExprEvaluator* const*, MemPool*, StringValue**, int*, int*);
template void Tuple::MaterializeExprs<false, true>(TupleRow*, const TupleDescriptor&,
    ScalarExprEvaluator* const*, MemPool*, StringValue**, int*, int*);
template void Tuple::MaterializeExprs<true, false>(TupleRow*, const TupleDescriptor&,
    ScalarExprEvaluator* const*, MemPool*, StringValue**, int*, int*);
template void Tuple::MaterializeExprs<true, true>(TupleRow*, const TupleDescriptor&,
    ScalarExprEvaluator* const*, MemPool*, StringValue**, int*, int*);
}