common/thrift/CMakeLists.txt

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


cmake_minimum_required(VERSION 2.6)

# Helper function to generate build rules.  For each input thrift file, this function will
# generate a rule that maps the input file to the output c++ file.
# Thrift will generate multiple output files for each input (including java files) and
# ideally, we'd specify all of the outputs for dependency tracking.
# Unfortunately, it's not easy to figure out all the output files without parsing the
# thrift input. (TODO: can thrift tells us what the java output files will be?)
# The list of output files is used for build dependency tracking so it's not necessary to
# capture all the output files.
#
# To call this function, pass it the output file list followed by the input thrift files:
#    i.e. THRIFT_GEN(OUTPUT_FILES, ${THRIFT_FILES})
#
# cmake seems to be case sensitive for some keywords. Changing the first IF check to lower
# case makes it not work.  TODO: investigate this
function(THRIFT_GEN VAR)
  IF (NOT ARGN)
    MESSAGE(SEND_ERROR "Error: THRIFT_GEN called without any src files")
    RETURN()
  ENDIF(NOT ARGN)

  set(${VAR})
  foreach(THRIFT_FILE ${ARGN})
    # Get full path
    get_filename_component(ABS_THRIFT_FILE ${THRIFT_FILE} ABSOLUTE)
    # Get basename without the file extension
    get_filename_component(THRIFT_FILE_WE ${THRIFT_FILE} NAME_WE)

    # All the output files we can determine based on filename.
    #   - Does not include .skeleton.cpp files
    #   - Does not include java output files
    set(OUTPUT_BE_FILE "${BE_OUTPUT_DIR}/gen-cpp/${THRIFT_FILE_WE}_types.cpp")
    set(OUTPUT_BE_FILE ${OUTPUT_BE_FILE} "${BE_OUTPUT_DIR}/gen-cpp/${THRIFT_FILE_WE}_types.h")
    set(OUTPUT_BE_FILE ${OUTPUT_BE_FILE} "${BE_OUTPUT_DIR}/gen-cpp/${THRIFT_FILE_WE}_constants.cpp")
    set(OUTPUT_BE_FILE ${OUTPUT_BE_FILE} "${BE_OUTPUT_DIR}/gen-cpp/${THRIFT_FILE_WE}_constants.h")
    list(APPEND ${VAR} ${OUTPUT_BE_FILE})

    # BeeswaxService thrift generation
    # It depends on hive_meta_store, which in turn depends on fb303.
    # The java dependency is handled by maven.
    # We need to generate C++ src file for the parent dependencies using the "-r" option.
    set(CPP_ARGS ${THRIFT_INCLUDE_DIR_OPTION}
        --gen cpp:moveable_types,no_default_operators -o ${BE_OUTPUT_DIR})
    IF (THRIFT_FILE STREQUAL "beeswax.thrift")
      set(CPP_ARGS -r ${CPP_ARGS})
    ENDIF(THRIFT_FILE STREQUAL "beeswax.thrift")

    IF (THRIFT_FILE STREQUAL ${TCLI_SERVICE_THRIFT} OR THRIFT_FILE STREQUAL "parquet.thrift" OR
        THRIFT_FILE STREQUAL "ImpalaService.thrift")
      # Do not generate Java HiveServer2 and Parquet files because we should use the jar
      # from Hive or Parquet.
      # Also do not generate ImpalaService.thrift because the generated code doesn't
      # compile with hive if the thrift version in hive is 0.9.0
      add_custom_command(
        OUTPUT ${OUTPUT_BE_FILE}
        COMMAND ${THRIFT_COMPILER} ${CPP_ARGS} ${THRIFT_FILE}
        COMMAND ${THRIFT_COMPILER} ${PYTHON_ARGS} ${THRIFT_FILE}
        DEPENDS ${ABS_THRIFT_FILE}
        COMMENT "Running thrift compiler on ${THRIFT_FILE}"
        VERBATIM
      )
    ELSE (THRIFT_FILE STREQUAL ${TCLI_SERVICE_THRIFT} OR THRIFT_FILE STREQUAL "parquet.thrift" OR
        THRIFT_FILE STREQUAL "ImpalaService.thrift")
      add_custom_command(
        OUTPUT ${OUTPUT_BE_FILE}
        COMMAND ${THRIFT_COMPILER} ${CPP_ARGS} ${THRIFT_FILE}
        COMMAND ${THRIFT_COMPILER} ${JAVA_FE_ARGS} ${THRIFT_FILE}
        COMMAND ${THRIFT_COMPILER} ${PYTHON_ARGS} ${THRIFT_FILE}
        DEPENDS ${ABS_THRIFT_FILE}
        COMMENT "Running thrift compiler on ${THRIFT_FILE}"
        VERBATIM
    )
    ENDIF (THRIFT_FILE STREQUAL ${TCLI_SERVICE_THRIFT} OR THRIFT_FILE STREQUAL "parquet.thrift" OR
        THRIFT_FILE STREQUAL "ImpalaService.thrift")
  endforeach(THRIFT_FILE)

  set(${VAR} ${${VAR}} PARENT_SCOPE)
endfunction(THRIFT_GEN)

function(THRIFT_GEN_DS VAR)
  IF (NOT ARGN)
    MESSAGE(SEND_ERROR "Error: THRIFT_GEN_DS called without any src files")
    RETURN()
  ENDIF(NOT ARGN)

  set(${VAR})
  foreach(THRIFT_FILE ${ARGN})
    get_filename_component(ABS_THRIFT_FILE ${THRIFT_FILE} ABSOLUTE)
    get_filename_component(THRIFT_FILE_WE ${THRIFT_FILE} NAME_WE)

    # Create a dummy marker file to track if Thrift compilation of the current file has
    # been run or not. This file is created only if Thrift compilation succeeds. Deletion
    # of the file will re-trigger Thrift compilation for the current file.
    set(OUTPUT_FILE_MESSAGE "CMake state file for Thrift compilation of"
                            "${ABS_THRIFT_FILE}, deletion of this file will cause"
                            "${THRIFT_FILE} to be recompiled during the next build")
    set(OUTPUT_DIR "${EXT_DS_OUTPUT_DIR}/../target/tmp/generated-sources/")
    set(OUTPUT_FILE "${OUTPUT_DIR}/${THRIFT_FILE_WE}.txt")
    list(APPEND ${VAR} ${OUTPUT_FILE})
    add_custom_command(
      OUTPUT ${OUTPUT_FILE}
      COMMAND ${THRIFT_COMPILER} ${JAVA_EXT_DS_ARGS} ${THRIFT_FILE} &&
              mkdir -p ${OUTPUT_DIR} && echo ${OUTPUT_FILE_MESSAGE} > ${OUTPUT_FILE}
      DEPENDS ${ABS_THRIFT_FILE}
      COMMENT "Running thrift compiler for ext-data-source on ${THRIFT_FILE}"
      VERBATIM
    )
  endforeach(THRIFT_FILE)
  set(${VAR} ${${VAR}} PARENT_SCOPE)
endfunction(THRIFT_GEN_DS)

# Helper function similar to THRIFT_GEN and THRIFT_GEN_DS, but specific to Thrift 11. For
# now the method only generates Python code. The generated Python code is not used by
# any other Impala module, it is merely meant to provide devs access to Thrift 11 Python
# generated code until a full upgrade from Thrift 0.9 to 0.11 has been completed. Once
# the complete upgrade to Thrift 11 has been complete (see IMPALA-7825), this method
# should be removed.
function(THRIFT11_GEN VAR)
  IF (NOT ARGN)
    MESSAGE(SEND_ERROR "Error: THRIFT11_GEN called without any src files")
    RETURN()
  ENDIF(NOT ARGN)

  set(${VAR})
  foreach(THRIFT_FILE ${ARGN})
    # Get full path
    get_filename_component(ABS_THRIFT__FILE ${THRIFT_FILE} ABSOLUTE)
    # Get basename without the file extension
    get_filename_component(THRIFT_FILE_WE ${THRIFT_FILE} NAME_WE)

    # This isn't the exact output file that's created, just a unique file
    set(OUTPUT_FILE "${THRIFT11_PYTHON_OUTPUT_DIR}/gen-py/${THRIFT_FILE_WE}/__init__.py")

    # We have to special case beeswax.thrift because its Python namespace is beeswaxd
    # instead of beeswax
    if (${THRIFT_FILE_WE} STREQUAL "beeswax")
      set(OUTPUT_FILE "${THRIFT11_PYTHON_OUTPUT_DIR}/gen-py/beeswaxd/__init__.py")
    endif(${THRIFT_FILE_WE} STREQUAL "beeswax")

    list(APPEND ${VAR} ${OUTPUT_FILE})
    add_custom_command(
      OUTPUT ${OUTPUT_FILE}
      COMMAND ${THRIFT11_COMPILER} ${THRIFT11_PYTHON_ARGS} ${THRIFT_FILE}
      DEPENDS ${ABS_THRIFT_FILE}
      COMMENT "Running thrift 11 compiler on ${THRIFT_FILE}"
      VERBATIM
    )
  endforeach(THRIFT_FILE)
  set(${VAR} ${${VAR}} PARENT_SCOPE)
endfunction(THRIFT11_GEN)

set(HIVE_THRIFT_SOURCE_DIR "hive-$ENV{IMPALA_HIVE_MAJOR_VERSION}-api")
set(TCLI_SERVICE_THRIFT "${HIVE_THRIFT_SOURCE_DIR}/TCLIService.thrift")
message("Using Thrift compiler: ${THRIFT_COMPILER}")
message("Using Thrift 11 compiler: ${THRIFT11_COMPILER}")
set(THRIFT_INCLUDE_DIR_OPTION -I ${THRIFT_CONTRIB_DIR} -I $ENV{HIVE_METASTORE_THRIFT_DIR}
  -I ${HIVE_THRIFT_SOURCE_DIR})
set(BE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/be/generated-sources)
set(FE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/fe/generated-sources)
# TODO: avoid duplicating generated java classes
set(EXT_DS_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/ext-data-source/api/generated-sources)
set(PYTHON_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/shell/)
set(THRIFT11_PYTHON_OUTPUT_DIR ${PYTHON_OUTPUT_DIR}/build/thrift-11-gen)
MESSAGE("Found output dir: " ${PYTHON_OUTPUT_DIR})
file(MAKE_DIRECTORY ${BE_OUTPUT_DIR})
file(MAKE_DIRECTORY ${FE_OUTPUT_DIR})
file(MAKE_DIRECTORY ${EXT_DS_OUTPUT_DIR})
file(MAKE_DIRECTORY ${PYTHON_OUTPUT_DIR})
file(MAKE_DIRECTORY ${HIVE_THRIFT_SOURCE_DIR})
file(MAKE_DIRECTORY ${THRIFT11_PYTHON_OUTPUT_DIR})

# Args passed to thrift for Java gen
set(JAVA_FE_ARGS ${THRIFT_INCLUDE_DIR_OPTION} --gen java -o ${FE_OUTPUT_DIR})
set(JAVA_EXT_DS_ARGS ${THRIFT_INCLUDE_DIR_OPTION} --gen java -o ${EXT_DS_OUTPUT_DIR})
set(PYTHON_ARGS ${THRIFT_INCLUDE_DIR_OPTION} -r --gen py -o ${PYTHON_OUTPUT_DIR})
set(THRIFT11_PYTHON_ARGS ${THRIFT_INCLUDE_DIR_OPTION} -r --gen py -o
                         ${THRIFT11_PYTHON_OUTPUT_DIR})

set (EXT_DATA_SRC_FILES
  ErrorCodes.thrift
  ExternalDataSource.thrift
  Data.thrift
  Status.thrift
  Types.thrift
)

set (SRC_FILES
  ErrorCodes.thrift
  beeswax.thrift
  BackendGflags.thrift
  CatalogInternalService.thrift
  CatalogObjects.thrift
  CatalogService.thrift
  DataSinks.thrift
  Descriptors.thrift
  ExecStats.thrift
  Frontend.thrift
  Exprs.thrift
  ExternalDataSource.thrift
  ImpalaInternalService.thrift
  ImpalaService.thrift
  JniCatalog.thrift
  LineageGraph.thrift
  Logging.thrift
  NetworkTest.thrift
  MetricDefs.thrift
  Metrics.thrift
  PlanNodes.thrift
  Planner.thrift
  Partitions.thrift
  parquet.thrift
  ResourceProfile.thrift
  Results.thrift
  RuntimeProfile.thrift
  StatestoreService.thrift
  Zip.thrift
  ${TCLI_SERVICE_THRIFT}
  ${EXT_DATA_SRC_FILES}
)

SET_SOURCE_FILES_PROPERTIES(Status.thrift PROPERTIES OBJECT_DEPENDS ErrorCodes.thrift)

add_custom_command(OUTPUT ErrorCodes.thrift
  COMMAND python generate_error_codes.py
  DEPENDS generate_error_codes.py)

add_custom_command(OUTPUT MetricDefs.thrift
  COMMAND python generate_metrics.py
  DEPENDS generate_metrics.py metrics.json)

# The thrift-generated java classes defined in TCLIService are also pulled into our build
# in the Hive jars that are downloaded via Maven. Hive2 moved the classes from
# org.apache.hive.service.cli.thrift to org.apache.hive.service.rpc.thrift. Impala calls
# various Hive methods that have these classes in the interface and if the packages don't
# match it won't compile.
add_custom_command(OUTPUT hive-2-api/TCLIService.thrift
  COMMAND sed
      's/namespace java org.apache.hive.service.cli.thrift/namespace java org.apache.hive.service.rpc.thrift/'
      hive-1-api/TCLIService.thrift > hive-2-api/TCLIService.thrift
  DEPENDS hive-1-api/TCLIService.thrift
)

add_custom_command(OUTPUT hive-3-api/TCLIService.thrift
  COMMAND sed
      's/namespace java org.apache.hive.service.cli.thrift/namespace java org.apache.hive.service.rpc.thrift/'
      hive-1-api/TCLIService.thrift > hive-3-api/TCLIService.thrift
  DEPENDS hive-1-api/TCLIService.thrift
)

# Create a build command for each of the thrift src files and generate
# a list of files they produce
THRIFT_GEN(THRIFT_ALL_FILES ${SRC_FILES})
THRIFT_GEN_DS(THRIFT_DATA_SRC_FILES ${EXT_DATA_SRC_FILES})
THRIFT11_GEN(THRIFT11_ALL_FILES ${SRC_FILES})

add_custom_target(thrift-generated-files-error DEPENDS ErrorCodes.thrift)
add_custom_target(thrift-generated-files-metrics DEPENDS MetricDefs.thrift)
add_custom_target(thrift-generated-files-tcli-service DEPENDS ${TCLI_SERVICE_THRIFT})

# Add a custom target that generates all the thrift files
add_custom_target(thrift-cpp ALL DEPENDS ${THRIFT_ALL_FILES})
add_dependencies(thrift-cpp thrift-generated-files-metrics thrift-generated-files-error
  thrift-generated-files-tcli-service)

add_custom_target(thrift-ext-data-src ALL DEPENDS ${THRIFT_DATA_SRC_FILES})
add_dependencies(thrift-ext-data-src thrift-cpp)

add_custom_target(thrift11-py ALL DEPENDS ${THRIFT11_ALL_FILES})
add_dependencies(thrift11-py thrift-ext-data-src)

# Combined target for all thrift dependencies
add_custom_target(thrift-deps ALL)
add_dependencies(thrift-deps thrift11-py)