[LIVY-754][THRIFT] Encode precision and scale for decimal type.
## What changes were proposed in this pull request?
When a `org.apache.livy.thriftserver.session.DataType.DECIMAL` is converted to a `org.apache.hive.service.rpc.thrift.TTypeDesc` for sending a Thrift response to a client request for result set metadata, the `TTypeDesc` contains a `TPrimitiveTypeEntry(TTypeId.DECIMAL_TYPE)` without `TTypeQualifiers` (which are needed to capture the precision and scale).
With this change, we include the qualifiers in the `TPrimitiveTypeEntry`. We use both the name and the `DataType` of a field type to construct the `TTypeDesc`. We are able to do this without changing the existing internal representation for data types because we can obtain the precision and scale from the name of the decimal type.
## How was this patch tested?
Use beeline to connect to the Thrift server. Do a select from a table with a column of decimal type.
Also extended an existing integration test.
Author: Wing Yew Poon <wypoon@cloudera.com>
Closes #288 from wypoon/LIVY-754.
diff --git a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
index 0f4c642..5c02159 100644
--- a/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
+++ b/thriftserver/server/src/main/scala/org/apache/livy/thriftserver/types/Schema.scala
@@ -17,7 +17,7 @@
package org.apache.livy.thriftserver.types
-import org.apache.hive.service.rpc.thrift.{TColumnDesc, TPrimitiveTypeEntry, TTableSchema, TTypeDesc, TTypeEntry, TTypeId}
+import org.apache.hive.service.rpc.thrift._
import org.apache.livy.thriftserver.session.DataType
@@ -41,7 +41,7 @@
case "float" => DataType.FLOAT
case "double" => DataType.DOUBLE
case "binary" => DataType.BINARY
- case _ if name.contains("decimal") => DataType.DECIMAL
+ case _ if name.startsWith("decimal") => DataType.DECIMAL
case "timestamp" => DataType.TIMESTAMP
case "date" => DataType.DATE
case _ => DataType.STRING
@@ -88,12 +88,12 @@
val tColumnDesc = new TColumnDesc
tColumnDesc.setColumnName(field.name)
tColumnDesc.setComment(field.comment)
- tColumnDesc.setTypeDesc(toTTypeDesc(field.fieldType.dataType))
+ tColumnDesc.setTypeDesc(toTTypeDesc(field.fieldType.name, field.fieldType.dataType))
tColumnDesc.setPosition(index)
tColumnDesc
}
- private def toTTypeDesc(dt: DataType): TTypeDesc = {
+ private def toTTypeDesc(name: String, dt: DataType): TTypeDesc = {
val typeId = dt match {
case DataType.BOOLEAN => TTypeId.BOOLEAN_TYPE
case DataType.BYTE => TTypeId.TINYINT_TYPE
@@ -109,9 +109,38 @@
case _ => TTypeId.STRING_TYPE
}
val primitiveEntry = new TPrimitiveTypeEntry(typeId)
+ if (dt == DataType.DECIMAL) {
+ val qualifiers = getDecimalQualifiers(name)
+ primitiveEntry.setTypeQualifiers(qualifiers)
+ }
val entry = TTypeEntry.primitiveEntry(primitiveEntry)
val desc = new TTypeDesc
desc.addToTypes(entry)
desc
}
+
+ private def getDecimalQualifiers(name: String): TTypeQualifiers = {
+ // name can be one of
+ // 1. decimal
+ // 2. decimal(p)
+ // 3. decimal(p, s)
+ val (precision, scale) =
+ if (name == "decimal") {
+ (10, 0)
+ } else {
+ val suffix = name.substring("decimal".length)
+ require(suffix.startsWith("(") && suffix.endsWith(")"),
+ name + " is not of the form decimal(<precision>,<scale>)")
+ val parts = suffix.substring(1, suffix.length - 1).split(",").map(_.trim.toInt)
+ (parts(0), parts.lift(1).getOrElse(0))
+ }
+ val qMap = new java.util.HashMap[String, TTypeQualifierValue]
+ val pVal = new TTypeQualifierValue
+ pVal.setI32Value(precision)
+ qMap.put(TCLIServiceConstants.PRECISION, pVal)
+ val sVal = new TTypeQualifierValue
+ sVal.setI32Value(scale)
+ qMap.put(TCLIServiceConstants.SCALE, sVal)
+ new TTypeQualifiers(qMap)
+ }
}
diff --git a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
index 19abb0d..44fd970 100644
--- a/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
+++ b/thriftserver/server/src/test/scala/org/apache/livy/thriftserver/ThriftServerSuites.scala
@@ -47,7 +47,9 @@
"cast('varchar_val' as varchar(20))," +
"cast('char_val' as char(20))," +
"cast('2018-08-06 09:11:15' as timestamp)," +
- "cast('2018-08-06' as date)")
+ "cast('2018-08-06' as date)," +
+ "cast(1234567890 as decimal(10))," +
+ "cast(1234567890 as decimal)")
val rsMetaData = resultSet.getMetaData()
@@ -73,6 +75,8 @@
assert(resultSet.getBigDecimal(7).doubleValue() == 7.7)
assert(rsMetaData.getColumnTypeName(7) == "decimal")
+ assert(rsMetaData.getPrecision(7) == 10)
+ assert(rsMetaData.getScale(7) == 1)
assert(resultSet.getBoolean(8) == true)
assert(rsMetaData.getColumnTypeName(8) == "boolean")
@@ -99,6 +103,16 @@
compareTo(Date.valueOf("2018-08-06")) == 0)
assert(rsMetaData.getColumnTypeName(14) == "date")
+ assert(resultSet.getBigDecimal(15).intValue() == 1234567890)
+ assert(rsMetaData.getColumnTypeName(15) == "decimal")
+ assert(rsMetaData.getPrecision(15) == 10)
+ assert(rsMetaData.getScale(15) == 0)
+
+ assert(resultSet.getBigDecimal(16).intValue() == 1234567890)
+ assert(rsMetaData.getColumnTypeName(16) == "decimal")
+ assert(rsMetaData.getPrecision(16) == 10)
+ assert(rsMetaData.getScale(16) == 0)
+
assert(!resultSet.next())
val resultSetWithNulls = statement.executeQuery(
