flink-libraries/flink-table/src/main/scala/org/apache/flink/table/api/scala/BatchTableEnvironment.scala

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.flink.table.api.scala

import org.apache.flink.annotation.VisibleForTesting

import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.io.CollectionInputFormat
import org.apache.flink.api.scala.{createTypeInformation, getCallLocationName}
import org.apache.flink.streaming.api.datastream.{DataStream => JDataStream}
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment, asScalaStream}
import org.apache.flink.table.api._
import org.apache.flink.table.api.functions.{AggregateFunction, TableFunction}
import org.apache.flink.table.expressions.Expression
import org.apache.flink.table.sources.RangeInputFormat
import org.apache.flink.table.typeutils.BaseRowTypeInfo

/**
  * The [[TableEnvironment]] for a Scala [[StreamExecutionEnvironment]].
  *
  * A TableEnvironment can be used to:
  * - convert a [[DataStream]] to a [[Table]]
  * - register a [[DataStream]] in the [[TableEnvironment]]'s catalog
  * - register a [[Table]] in the [[TableEnvironment]]'s catalog
  * - scan a registered table to obtain a [[Table]]
  * - specify a SQL query on registered tables to obtain a [[Table]]
  * - convert a [[Table]] into a [[DataStream]]
  * - explain the AST and execution plan of a [[Table]]
  *
  * @param execEnv The Scala [[StreamExecutionEnvironment]] of the TableEnvironment.
  * @param config  The configuration of the TableEnvironment.
  */
class BatchTableEnvironment(
    execEnv: StreamExecutionEnvironment,
    config: TableConfig)
  extends org.apache.flink.table.api.BatchTableEnvironment(
    execEnv.getWrappedStreamExecutionEnvironment,
    config) {

  /**
    * Converts the given [[DataStream]] into a [[Table]].
    *
    * The field names of the [[Table]] are automatically derived from the type of the
    * [[DataStream]].
    *
    * @param boundedStream The [[DataStream]] to be converted.
    * @tparam T The type of the [[DataStream]].
    * @return The converted [[Table]].
    */
  def fromBoundedStream[T](boundedStream: DataStream[T]): Table = {

    val name = createUniqueTableName()
    registerBoundedStreamInternal(name, boundedStream.javaStream, false)
    scan(name)
  }

  /**
    * Converts the given [[DataStream]] into a [[Table]].
    *
    * The field names of the [[Table]] are automatically derived from the type of the
    * [[DataStream]].
    *
    * @param boundedStream The [[DataStream]] to be converted.
    * @param fieldNullables The field isNullables attributes of boundedStream.
    * @tparam T The type of the [[DataStream]].
    * @return The converted [[Table]].
    */
  def fromBoundedStream[T](
      boundedStream: DataStream[T],
      fieldNullables: Array[Boolean]): Table = {

    val name = createUniqueTableName()
    registerBoundedStreamInternal(name, boundedStream.javaStream, fieldNullables, false)
    scan(name)
  }

  /**
    * Converts the given [[DataStream]] into a [[Table]] with specified field names.
    *
    * Example:
    *
    * {{{
    *   val stream: BoundedStream[(String, Long)] = ...
    *   val tab: Table = tableEnv.fromBoundedStream(stream, 'a, 'b)
    * }}}
    *
    * @param boundedStream The [[DataStream]] to be converted.
    * @param fields         The field names of the resulting [[Table]].
    * @tparam T The type of the [[DataStream]].
    * @return The converted [[Table]].
    */
  def fromBoundedStream[T](boundedStream: DataStream[T], fields: Expression*): Table = {

    val name = createUniqueTableName()
    registerBoundedStreamInternal(name, boundedStream.javaStream, fields.toArray, false)
    scan(name)
  }

  /**
    * Converts the given [[DataStream]] into a [[Table]] with specified field names.
    *
    * Example:
    *
    * {{{
    *   val stream: DataStream[(String, Long)] = ...
    *   val fieldNullables: Array[Boolean] = ...
    *   val tab: Table = tableEnv.fromBoundedStream(stream, fieldNullables, 'a, 'b)
    * }}}
    *
    * @param boundedStream The [[DataStream]] to be converted.
    * @param fieldNullables The field isNullables attributes of boundedStream.
    * @param fields         The field names of the resulting [[Table]].
    * @tparam T The type of the [[DataStream]].
    * @return The converted [[Table]].
    */
  def fromBoundedStream[T](
      boundedStream: DataStream[T],
      fieldNullables: Array[Boolean],
      fields: Expression*): Table = {

    val name = createUniqueTableName()
    registerBoundedStreamInternal(name, boundedStream.javaStream,
      fields.toArray, fieldNullables, false)
    scan(name)
  }

  /**
    * Converts the given [[DataStream]] into a [[Table]] with specified field names.
    *
    * Example:
    *
    * {{{
    *   val stream: DataStream[(String, Long)] = ...
    *   val tab: Table = tableEnv.fromBoundedStream(stream, 'a, 'b)
    * }}}
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to be converted.
    * @param fields         The field names of the resulting [[Table]].
    * @tparam T The type of the [[DataStream]].
    * @return The converted [[Table]].
    */
  def fromBoundedStream[T](name: String, boundedStream: DataStream[T], fields: Expression*):
  Table = {
    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream, fields.toArray, false)
    scan(name)
  }

  /**
    * Converts the given [[DataStream]] into a [[Table]] with specified field names.
    *
    * Example:
    *
    * {{{
    *   val stream: DataStream[(String, Long)] = ...
    *   val fieldNullables: Array[Boolean] = ...
    *   val tab: Table = tableEnv.fromBoundedStream(stream, fieldNullables, 'a, 'b)
    * }}}
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to be converted.
    * @param fieldNullables The field isNullables attributes of boundedStream.
    * @param fields         The field names of the resulting [[Table]].
    * @tparam T The type of the [[DataStream]].
    * @return The converted [[Table]].
    */
  def fromBoundedStream[T](
      name: String,
      boundedStream: DataStream[T],
      fieldNullables: Array[Boolean],
      fields: Expression*):
  Table = {
    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream,
      fields.toArray, fieldNullables, false)
    scan(name)
  }

  /**
    * Registers the given [[DataStream]] as table in the
    * [[TableEnvironment]]'s catalog.
    * Registered tables can be referenced in SQL queries.
    *
    * The field names of the [[Table]] are automatically derived
    * from the type of the [[DataStream]].
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to register.
    * @tparam T The type of the [[DataStream]] to register.
    */
  def registerBoundedStream[T](name: String, boundedStream: DataStream[T]): Unit = {

    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream, false)
  }


  /**
    * Registers or replace the given [[DataStream]] as table in the
    * [[TableEnvironment]]'s catalog.
    * Registered tables can be referenced in SQL queries.
    *
    * The field names of the [[Table]] are automatically derived
    * from the type of the [[DataStream]].
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to register.
    * @tparam T The type of the [[DataStream]] to register.
    */
  def registerOrReplaceBoundedStream[T](name: String, boundedStream: DataStream[T]): Unit = {

    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream, true)
  }

  /**
    * Registers the given [[DataStream]] as table in the
    * [[TableEnvironment]]'s catalog.
    * Registered tables can be referenced in SQL queries.
    *
    * The field names of the [[Table]] are automatically derived
    * from the type of the [[DataStream]].
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to register.
    * @param fieldNullables The field isNullables attributes of boundedStream.
    * @tparam T The type of the [[DataStream]] to register.
    */
  def registerBoundedStream[T](
      name: String,
      boundedStream: DataStream[T],
      fieldNullables: Array[Boolean]): Unit = {

    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream, fieldNullables, false)
  }

  /**
    * Registers or replace the given [[DataStream]] as table in the
    * [[TableEnvironment]]'s catalog.
    * Registered tables can be referenced in SQL queries.
    *
    * The field names of the [[Table]] are automatically derived
    * from the type of the [[DataStream]].
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to register.
    * @param fieldNullables The field isNullables attributes of boundedStream.
    * @tparam T The type of the [[DataStream]] to register.
    */
  def registerOrReplaceBoundedStream[T](
                                        name: String,
                                        boundedStream: DataStream[T],
                                        fieldNullables: Array[Boolean]): Unit = {

    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream, fieldNullables, true)
  }

  /**
    * Registers the given [[DataStream]] as table with specified field names in the
    * [[TableEnvironment]]'s catalog.
    * Registered tables can be referenced in SQL queries.
    *
    * Example:
    *
    * {{{
    *   val set: DataStream[(String, Long)] = ...
    *   tableEnv.registerBoundedStream("myTable", set, 'a, 'b)
    * }}}
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to register.
    * @param fields         The field names of the registered table.
    * @tparam T The type of the [[DataStream]] to register.
    */
  def registerBoundedStream[T](
      name: String,
      boundedStream: DataStream[T],
      fields: Expression*): Unit = {

    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream, fields.toArray, false)
  }

  /**
    * Registers or replace the given [[DataStream]] as table with specified field names in the
    * [[TableEnvironment]]'s catalog.
    * Registered tables can be referenced in SQL queries.
    *
    * Example:
    *
    * {{{
    *   val set: DataStream[(String, Long)] = ...
    *   tableEnv.registerBoundedStream("myTable", set, 'a, 'b)
    * }}}
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to register.
    * @param fields         The field names of the registered table.
    * @tparam T The type of the [[DataStream]] to register.
    */
  def registerOrReplaceBoundedStream[T](
                                        name: String,
                                        boundedStream: DataStream[T],
                                        fields: Expression*): Unit = {

    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream, fields.toArray, true)
  }

  /**
    * Registers the given [[DataStream]] as table with specified field names in the
    * [[TableEnvironment]]'s catalog.
    * Registered tables can be referenced in SQL queries.
    *
    * Example:
    *
    * {{{
    *   val set: BoundedStream[(String, Long)] = ...
    *   val fieldNullables: Array[Boolean] = ...
    *   tableEnv.registerBoundedStream("myTable", set, fieldNullables, 'a, 'b)
    * }}}
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to register.
    * @param fieldNullables The field isNullables attributes of boundedStream.
    * @param fields         The field names of the registered table.
    * @tparam T The type of the [[DataStream]] to register.
    */
  def registerBoundedStream[T](
      name: String,
      boundedStream: DataStream[T],
      fieldNullables: Array[Boolean],
      fields: Expression*): Unit = {

    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream,
      fields.toArray, fieldNullables, false)
  }

  /**
    * Registers the given [[DataStream]] as table with specified field names in the
    * [[TableEnvironment]]'s catalog.
    * Registered tables can be referenced in SQL queries.
    *
    * Example:
    *
    * {{{
    *   val set: BoundedStream[(String, Long)] = ...
    *   val fieldNullables: Array[Boolean] = ...
    *   tableEnv.registerBoundedStream("myTable", set, fieldNullables, 'a, 'b)
    * }}}
    *
    * @param name           The name under which the [[DataStream]] is registered in the
    *                       catalog.
    * @param boundedStream The [[DataStream]] to register.
    * @param fieldNullables The field isNullables attributes of boundedStream.
    * @param fields         The field names of the registered table.
    * @tparam T The type of the [[DataStream]] to register.
    */
  def registerOrReplaceBoundedStream[T](
                                        name: String,
                                        boundedStream: DataStream[T],
                                        fieldNullables: Array[Boolean],
                                        fields: Expression*): Unit = {

    checkValidTableName(name)
    registerBoundedStreamInternal(name, boundedStream.javaStream,
      fields.toArray, fieldNullables, true)
  }

  /**
    * Translates a [[Table]] into a bounded [[DataStream]] of specific type.
    *
    * The fields of the [[Table]] are mapped to [[DataStream]] fields as follows:
    * - [[org.apache.flink.types.Row]] and [[org.apache.flink.api.java.tuple.Tuple]]
    * types: Fields are mapped by position, field types must match.
    * - POJO [[DataStream]] types: Fields are mapped by field name, field types must match.
    *
    * @param table The [[Table]] to convert.
    * @tparam T The type of the [[DataStream]].
    * @return The generated [[DataStream]] operators translated by [[Table]].
    */
  def toBoundedStream[T: TypeInformation](table: Table): DataStream[T] = {
    val returnType = createTypeInformation[T]
    TableEnvironment.validateType(returnType)
    val javaStream: JDataStream[T] = translateToDataStream(table, returnType)
    asScalaStream(javaStream)
  }

  /**
    * Registers the given [[Iterable]] as table in the
    * [[TableEnvironment]]'s catalog.
    *
    * @param tableName name of table.
    * @param data The [[Iterable]] to be converted.
    * @param fields field names expressions, eg: 'a, 'b, 'c
    * @tparam T The type of the [[Iterable]].
    * @return The converted [[Table]].
    */
  def registerCollection[T : ClassTag : TypeInformation](
    tableName: String, data: Iterable[T], fields: Expression*): Unit = {
    val typeInfo = implicitly[TypeInformation[T]]
    val fieldArray = if (fields == null || fields.isEmpty) null else fields.toArray
    registerCollection(tableName, data, typeInfo, null, fieldArray)
  }

  /**
    * Registers the given [[Iterable]] as table in the
    * [[TableEnvironment]]'s catalog.
    *
    * @param tableName name of table.
    * @param data The [[Iterable]] to be converted.
    * @param fieldNullables The field isNullables attributes of data.
    * @param fields field names, eg: 'a, 'b, 'c
    * @tparam T The type of the [[Iterable]].
    * @return The converted [[Table]].
    */
  def registerCollection[T : ClassTag : TypeInformation](tableName: String, data: Iterable[T],
    fieldNullables: Iterable[Boolean], fields: Expression*): Unit = {
    val typeInfo = implicitly[TypeInformation[T]]
    val fieldArray = if (fields == null || fields.isEmpty) null else fields.toArray
    registerCollection(tableName, data, typeInfo, null, fieldArray)
  }

  /**
    * Registers the given [[Iterable]] as table in the
    * [[TableEnvironment]]'s catalog.
    *
    * @param tableName name of table.
    * @param data The [[Iterable]] to be converted.
    * @param information information of [[Iterable]].
    * @param fields field names expressions, eg: 'a, 'b, 'c
    * @tparam T The type of the [[Iterable]].
    * @return The converted [[Table]].
    */
  def registerCollection[T](tableName: String, data: Iterable[T], information: TypeInformation[T],
    fields: Expression*): Unit = {
    val fieldArray = if (fields == null || fields.isEmpty) null else fields.toArray
    registerCollection(tableName, data, information, null, fieldArray)
  }

  /**
    * Registers the given [[Iterable]] as table in the
    * [[TableEnvironment]]'s catalog.
    *
    * @param tableName name of table.
    * @param data The [[Iterable]] to be converted.
    * @param information information of [[Iterable]].
    * @param fieldNullables The field isNullables attributes of data.
    * @param fields field name expressions, eg: 'a, 'b, 'c
    * @tparam T The type of the [[Iterable]].
    * @return The converted [[Table]].
    */
  def registerCollection[T](
    tableName: String,
    data: Iterable[T],
    information: TypeInformation[T],
    fieldNullables: Iterable[Boolean],
    fields: Expression*): Unit = {
    val fieldArray = if (fields == null || fields.isEmpty) null else fields.toArray
    registerCollection(tableName, data, information, fieldNullables, fieldArray)
  }

  /**
    * Registers the given [[Iterable]] as table in the
    * [[TableEnvironment]]'s catalog.
    *
    * @param tableName name of table.
    * @param data The [[Iterable]] to be converted.
    * @param typeInfo information of [[Iterable]].
    * @param fieldNullables The field isNullables attributes of data.
    * @param fields field name expressions, eg: 'a, 'b, 'c
    * @tparam T The type of the [[Iterable]].
    * @return The converted [[Table]].
    */
  @VisibleForTesting
  private [table] def registerCollection[T](
    tableName: String,
    data: Iterable[T],
    typeInfo: TypeInformation[T],
    fieldNullables: Iterable[Boolean],
    fields: Array[Expression]): Unit = {
    val boundedStream = streamEnv.createInput(new CollectionInputFormat[T](
      data.asJavaCollection,
      typeInfo.createSerializer(execEnv.getConfig)),
      typeInfo, tableName)
    boundedStream.forceNonParallel()
    (fields == null, fieldNullables == null) match {
      case (true, true) => registerBoundedStreamInternal(tableName, boundedStream, false)
      case (false, true) => registerBoundedStreamInternal(tableName, boundedStream, fields, false)
      case (false, false) => registerBoundedStreamInternal(tableName, boundedStream, fields,
        fieldNullables.toArray, false)
      case (true, false) => throw new IllegalArgumentException("Can not register collection with" +
        "empty field names while fieldNullables non empty.")
    }
  }

  /**
    * Create a [[Table]] from sequence of elements. Typical, user can pass in a sequence of tuples,
    * the table schema type would be inferred from the tuple type: e.g.
    * {{{
    *   tEnv.fromElements((1, 2, "abc"), (3, 4, "def"))
    * }}}
    * Then the schema type would be (_1:int, _2:int, _3:varchar)
    *
    * Caution that use must pass a ''Scala'' type data elements, or the inferred type
    * would be unexpected.
    *
    * @param data row data sequence
    * @tparam T row data class type
    * @return table from the data with default fields names
    */
  def fromElements[T: ClassTag : TypeInformation](data: T*): Table = {
    require(data != null, "Data must not be null.")
    val typeInfo = implicitly[TypeInformation[T]]
    fromCollection(data)(implicitly[ClassTag[T]], typeInfo)
  }

  /**
    * Create a [[Table]] from a scala [[Iterable]]. The default fields names
    * would be like _1, _2, _3 and so on. The table schema type would be inferred from the
    * [[Iterable]] element type.
    */
  def fromCollection[T: ClassTag : TypeInformation](data: Iterable[T]): Table = {
    val typeInfo = implicitly[TypeInformation[T]]
    fromCollection(null, data, typeInfo, null)
  }

  /**
    * Create a [[Table]] from a scala [[Iterable]]. The table schema type would be inferred
    * from the [[Iterable]] element type.
    */
  def fromCollection[T: ClassTag : TypeInformation](data: Iterable[T],
    fields: Expression*): Table = {
    require(data != null, "Data must not be null.")
    val typeInfo = implicitly[TypeInformation[T]]
    val fieldArray = if (fields == null || fields.isEmpty) null else fields.toArray
    fromCollection(null, data, typeInfo, fieldArray)
  }

  /**
    * Create a [[Table]] from a scala [[Iterable]]. The table schema type would be inferred
    * from the [[Iterable]] element type.
    *
    * Only used for testing now.
    */
  @VisibleForTesting
  private [table] def fromCollection[T: ClassTag : TypeInformation](data: Iterable[T],
    fields: Array[Expression]): Table = {
    require(data != null, "Data must not be null.")
    val typeInfo = implicitly[TypeInformation[T]]
    fromCollection(null, data, typeInfo, fields)
  }

  /**
    * Create a [[Table]] from a scala [[Iterable]]. Would infer table schema from the passed in
    * typeInfo.
    */
  def fromCollection[T](data: Iterable[T], typeInfo: TypeInformation[T],
    fields: Expression*): Table = {
    val fieldArray = if (fields == null || fields.isEmpty) null else fields.toArray
    fromCollection(null, data, typeInfo, fieldArray)
  }

  /**
    * Create a [[Table]] from a scala [[Iterable]]. Would infer table schema from the passed in
    * typeInfo.
    */
  @VisibleForTesting
  private [table] def fromCollection[T](
    tableName: String,
    data: Iterable[T],
    typeInfo: TypeInformation[T],
    fields: Array[Expression]): Table = {
    CollectionInputFormat.checkCollection(data.asJavaCollection, typeInfo.getTypeClass)
    val boundedStream = streamEnv.createInput(new CollectionInputFormat[T](
      data.asJavaCollection,
      typeInfo.createSerializer(execEnv.getConfig)),
      typeInfo, getCallLocationName())
    boundedStream.setParallelism(1)
    val name = if (tableName == null) createUniqueTableName() else tableName
    if (fields == null) {
      registerBoundedStreamInternal(name, boundedStream, false)
    } else {
      registerBoundedStreamInternal(name, boundedStream, fields, false)
    }
    scan(name)
  }

  /**
    * Creates a [[Table]] with a single `DataTypes.Long` column named `id`, containing elements
    * in a range from `start` to `end` (exclusive) with step value 1.
    */
  def range(start: Long, end: Long): Table = {
    val typeInfo = new BaseRowTypeInfo(Types.LONG)
    val boundedStream = streamEnv.createInput(new RangeInputFormat(start, end),
      typeInfo, getCallLocationName())
    fromBoundedStream(new DataStream(boundedStream), 'id)
  }

  /**
    * Registers an [[AggregateFunction]] under a unique name in the TableEnvironment's catalog.
    * Registered functions can be referenced in Table API and SQL queries.
    *
    * @param name The name under which the function is registered.
    * @param f The AggregateFunction to register.
    * @tparam T The type of the output value.
    * @tparam ACC The type of aggregate accumulator.
    */
  def registerFunction[T: TypeInformation, ACC: TypeInformation](
    name: String, f: AggregateFunction[T, ACC]): Unit = {
    registerAggregateFunctionInternal[T, ACC](name, f)
  }

  /**
    * Registers a [[TableFunction]] under a unique name in the TableEnvironment's catalog.
    * Registered functions can be referenced in Table API and SQL queries.
    *
    * @param name The name under which the function is registered.
    * @param tf The TableFunction to register.
    * @tparam T The type of the output row.
    */
  def registerFunction[T: TypeInformation](name: String, tf: TableFunction[T]): Unit = {
    registerTableFunctionInternal[T](name, tf)
  }
}