| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| package hivemall.utils.hadoop; |
| |
| import static hivemall.HivemallConstants.BIGINT_TYPE_NAME; |
| import static hivemall.HivemallConstants.BINARY_TYPE_NAME; |
| import static hivemall.HivemallConstants.BOOLEAN_TYPE_NAME; |
| import static hivemall.HivemallConstants.DECIMAL_TYPE_NAME; |
| import static hivemall.HivemallConstants.DOUBLE_TYPE_NAME; |
| import static hivemall.HivemallConstants.FLOAT_TYPE_NAME; |
| import static hivemall.HivemallConstants.INT_TYPE_NAME; |
| import static hivemall.HivemallConstants.SMALLINT_TYPE_NAME; |
| import static hivemall.HivemallConstants.STRING_TYPE_NAME; |
| import static hivemall.HivemallConstants.TINYINT_TYPE_NAME; |
| import static hivemall.HivemallConstants.VOID_TYPE_NAME; |
| |
| import java.nio.charset.StandardCharsets; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.BitSet; |
| import java.util.Collections; |
| import java.util.List; |
| import java.util.Properties; |
| |
| import javax.annotation.Nonnull; |
| import javax.annotation.Nullable; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.hive.ql.exec.UDFArgumentException; |
| import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; |
| import org.apache.hadoop.hive.ql.metadata.HiveException; |
| import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject; |
| import org.apache.hadoop.hive.serde2.SerDeException; |
| import org.apache.hadoop.hive.serde2.io.ByteWritable; |
| import org.apache.hadoop.hive.serde2.io.DoubleWritable; |
| import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable; |
| import org.apache.hadoop.hive.serde2.io.ShortWritable; |
| import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef; |
| import org.apache.hadoop.hive.serde2.lazy.LazyDouble; |
| import org.apache.hadoop.hive.serde2.lazy.LazyInteger; |
| import org.apache.hadoop.hive.serde2.lazy.LazyLong; |
| import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; |
| import org.apache.hadoop.hive.serde2.lazy.LazyString; |
| import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory; |
| import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector; |
| import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray; |
| import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap; |
| import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; |
| import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; |
| import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; |
| import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; |
| import org.apache.hadoop.hive.serde2.objectinspector.StandardConstantListObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.StructField; |
| import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; |
| import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantStringObjectInspector; |
| import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo; |
| import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; |
| import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; |
| import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; |
| import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; |
| import org.apache.hadoop.io.BooleanWritable; |
| import org.apache.hadoop.io.FloatWritable; |
| import org.apache.hadoop.io.IntWritable; |
| import org.apache.hadoop.io.LongWritable; |
| import org.apache.hadoop.io.Text; |
| import org.apache.hadoop.io.Writable; |
| |
| public final class HiveUtils { |
| |
| private HiveUtils() {} |
| |
| public static int parseInt(@Nonnull final Object o) { |
| if (o instanceof Integer) { |
| return ((Integer) o).intValue(); |
| } |
| if (o instanceof IntWritable) { |
| return ((IntWritable) o).get(); |
| } |
| if (o instanceof LongWritable) { |
| long l = ((LongWritable) o).get(); |
| if (l > 0x7fffffffL) { |
| throw new IllegalArgumentException( |
| "feature index must be less than " + Integer.MAX_VALUE + ", but was " + l); |
| } |
| return (int) l; |
| } |
| String s = o.toString(); |
| return Integer.parseInt(s); |
| } |
| |
| @Nullable |
| public static Writable copyToWritable(@Nullable final Object o, |
| @Nonnull final PrimitiveObjectInspector poi) { |
| if (o == null) { |
| return null; |
| } |
| Object copied = poi.copyObject(o); |
| Object result = poi.getPrimitiveWritableObject(copied); |
| return (Writable) result; |
| } |
| |
| public static Text asText(@Nullable final Object o) { |
| if (o == null) { |
| return null; |
| } |
| if (o instanceof Text) { |
| return (Text) o; |
| } |
| if (o instanceof LazyString) { |
| LazyString l = (LazyString) o; |
| return l.getWritableObject(); |
| } |
| if (o instanceof String) { |
| String s = (String) o; |
| return new Text(s); |
| } |
| String s = o.toString(); |
| return new Text(s); |
| } |
| |
| public static int asJavaInt(@Nullable final Object o, final int nullValue) { |
| if (o == null) { |
| return nullValue; |
| } |
| return asJavaInt(o); |
| } |
| |
| public static int asJavaInt(@Nullable final Object o) { |
| if (o == null) { |
| throw new IllegalArgumentException(); |
| } |
| if (o instanceof Integer) { |
| return ((Integer) o).intValue(); |
| } |
| if (o instanceof LazyInteger) { |
| IntWritable i = ((LazyInteger) o).getWritableObject(); |
| return i.get(); |
| } |
| if (o instanceof IntWritable) { |
| return ((IntWritable) o).get(); |
| } |
| String s = o.toString(); |
| return Integer.parseInt(s); |
| } |
| |
| public static double asJavaDouble(@Nullable final Object o) { |
| if (o == null) { |
| throw new IllegalArgumentException(); |
| } |
| if (o instanceof Double) { |
| return ((Double) o).doubleValue(); |
| } |
| if (o instanceof LazyDouble) { |
| DoubleWritable d = ((LazyDouble) o).getWritableObject(); |
| return d.get(); |
| } |
| if (o instanceof DoubleWritable) { |
| return ((DoubleWritable) o).get(); |
| } |
| String s = o.toString(); |
| return Double.parseDouble(s); |
| } |
| |
| @Nullable |
| public static List<String> asStringList(@Nonnull final DeferredObject arg, |
| @Nonnull final ListObjectInspector listOI) throws HiveException { |
| Object argObj = arg.get(); |
| if (argObj == null) { |
| return null; |
| } |
| List<?> data = listOI.getList(argObj); |
| int size = data.size(); |
| if (size == 0) { |
| return Collections.emptyList(); |
| } |
| final String[] ary = new String[size]; |
| for (int i = 0; i < size; i++) { |
| Object o = data.get(i); |
| if (o != null) { |
| ary[i] = o.toString(); |
| } |
| } |
| return Arrays.asList(ary); |
| } |
| |
| @Nullable |
| public static String[] asStringArray(@Nonnull final DeferredObject arg, |
| @Nonnull final ListObjectInspector listOI) throws HiveException { |
| Object argObj = arg.get(); |
| if (argObj == null) { |
| return null; |
| } |
| List<?> data = listOI.getList(argObj); |
| final int size = data.size(); |
| final String[] arr = new String[size]; |
| for (int i = 0; i < size; i++) { |
| Object o = data.get(i); |
| if (o != null) { |
| arr[i] = o.toString(); |
| } |
| } |
| return arr; |
| } |
| |
| @Nonnull |
| public static StructObjectInspector asStructOI(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| if (oi.getCategory() != Category.STRUCT) { |
| throw new UDFArgumentException("Expected Struct OI but got: " + oi.getTypeName()); |
| } |
| return (StructObjectInspector) oi; |
| } |
| |
| public static boolean isPrimitiveOI(@Nonnull final ObjectInspector oi) { |
| return oi.getCategory() == Category.PRIMITIVE; |
| } |
| |
| public static boolean isStructOI(@Nonnull final ObjectInspector oi) { |
| return oi.getCategory() == Category.STRUCT; |
| } |
| |
| public static boolean isVoidOI(@Nonnull final ObjectInspector oi) { |
| String typeName = oi.getTypeName(); |
| return VOID_TYPE_NAME.equals(typeName); |
| } |
| |
| public static boolean isStringOI(@Nonnull final ObjectInspector oi) { |
| String typeName = oi.getTypeName(); |
| return STRING_TYPE_NAME.equals(typeName); |
| } |
| |
| public static boolean isIntOI(@Nonnull final ObjectInspector oi) { |
| String typeName = oi.getTypeName(); |
| return INT_TYPE_NAME.equals(typeName); |
| } |
| |
| public static boolean isBigIntOI(@Nonnull final ObjectInspector oi) { |
| String typeName = oi.getTypeName(); |
| return BIGINT_TYPE_NAME.equals(typeName); |
| } |
| |
| public static boolean isBooleanOI(@Nonnull final ObjectInspector oi) { |
| String typeName = oi.getTypeName(); |
| return BOOLEAN_TYPE_NAME.equals(typeName); |
| } |
| |
| public static boolean isBinaryOI(@Nonnull final ObjectInspector oi) { |
| String typeName = oi.getTypeName(); |
| return BINARY_TYPE_NAME.equals(typeName); |
| } |
| |
| public static boolean isNumberOI(@Nonnull final ObjectInspector argOI) { |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| return false; |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case SHORT: |
| case INT: |
| case LONG: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| case BYTE: |
| //case TIMESTAMP: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| public static boolean isIntegerOI(@Nonnull final ObjectInspector argOI) { |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| return false; |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case INT: |
| case SHORT: |
| case LONG: |
| case BYTE: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| public static boolean isListOI(@Nonnull final ObjectInspector oi) { |
| Category category = oi.getCategory(); |
| return category == Category.LIST; |
| } |
| |
| public static boolean isStringListOI(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| Category category = oi.getCategory(); |
| if (category != Category.LIST) { |
| throw new UDFArgumentException("Expected List OI but was: " + oi); |
| } |
| ListObjectInspector listOI = (ListObjectInspector) oi; |
| return isStringOI(listOI.getListElementObjectInspector()); |
| } |
| |
| public static boolean isMapOI(@Nonnull final ObjectInspector oi) { |
| return oi.getCategory() == Category.MAP; |
| } |
| |
| public static boolean isNumberListOI(@Nonnull final ObjectInspector oi) { |
| return isListOI(oi) |
| && isNumberOI(((ListObjectInspector) oi).getListElementObjectInspector()); |
| } |
| |
| public static boolean isNumberListListOI(@Nonnull final ObjectInspector oi) { |
| return isListOI(oi) |
| && isNumberListOI(((ListObjectInspector) oi).getListElementObjectInspector()); |
| } |
| |
| public static boolean isConstListOI(@Nonnull final ObjectInspector oi) { |
| return ObjectInspectorUtils.isConstantObjectInspector(oi) && isListOI(oi); |
| } |
| |
| public static boolean isConstStringListOI(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| if (!isConstListOI(oi)) { |
| return false; |
| } |
| ListObjectInspector listOI = (ListObjectInspector) oi; |
| return isStringOI(listOI.getListElementObjectInspector()); |
| } |
| |
| public static boolean isConstString(@Nonnull final ObjectInspector oi) { |
| return ObjectInspectorUtils.isConstantObjectInspector(oi) && isStringOI(oi); |
| } |
| |
| public static boolean isConstInt(@Nonnull final ObjectInspector oi) { |
| return ObjectInspectorUtils.isConstantObjectInspector(oi) && isIntOI(oi); |
| } |
| |
| public static boolean isConstInteger(@Nonnull final ObjectInspector oi) { |
| return ObjectInspectorUtils.isConstantObjectInspector(oi) && isIntegerOI(oi); |
| } |
| |
| public static boolean isConstBoolean(@Nonnull final ObjectInspector oi) { |
| return ObjectInspectorUtils.isConstantObjectInspector(oi) && isBooleanOI(oi); |
| } |
| |
| public static boolean isPrimitiveTypeInfo(@Nonnull TypeInfo typeInfo) { |
| return typeInfo.getCategory() == ObjectInspector.Category.PRIMITIVE; |
| } |
| |
| public static boolean isStructTypeInfo(@Nonnull TypeInfo typeInfo) { |
| return typeInfo.getCategory() == ObjectInspector.Category.STRUCT; |
| } |
| |
| public static boolean isNumberTypeInfo(@Nonnull TypeInfo typeInfo) { |
| if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) { |
| return false; |
| } |
| switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) { |
| case BYTE: |
| case SHORT: |
| case INT: |
| case LONG: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| public static boolean isBooleanTypeInfo(@Nonnull TypeInfo typeInfo) { |
| if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) { |
| return false; |
| } |
| switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) { |
| case BOOLEAN: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| public static boolean isIntegerTypeInfo(@Nonnull TypeInfo typeInfo) { |
| if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) { |
| return false; |
| } |
| switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) { |
| case BYTE: |
| case SHORT: |
| case INT: |
| case LONG: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| public static boolean isIntTypeInfo(@Nonnull TypeInfo typeInfo) { |
| if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) { |
| return false; |
| } |
| return ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.INT; |
| } |
| |
| public static boolean isFloatingPointTypeInfo(@Nonnull TypeInfo typeInfo) { |
| if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) { |
| return false; |
| } |
| switch (((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()) { |
| case DOUBLE: |
| case FLOAT: |
| case DECIMAL: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| public static boolean isStringTypeInfo(@Nonnull TypeInfo typeInfo) { |
| if (typeInfo.getCategory() != ObjectInspector.Category.PRIMITIVE) { |
| return false; |
| } |
| return ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() == PrimitiveCategory.STRING; |
| } |
| |
| public static boolean isListTypeInfo(@Nonnull TypeInfo typeInfo) { |
| return typeInfo.getCategory() == Category.LIST; |
| } |
| |
| public static boolean isFloatingPointListTypeInfo(@Nonnull TypeInfo typeInfo) { |
| if (typeInfo.getCategory() != Category.LIST) { |
| return false; |
| } |
| TypeInfo elemTypeInfo = ((ListTypeInfo) typeInfo).getListElementTypeInfo(); |
| return isFloatingPointTypeInfo(elemTypeInfo); |
| } |
| |
| @Nonnull |
| public static ListTypeInfo asListTypeInfo(@Nonnull TypeInfo typeInfo) |
| throws UDFArgumentException { |
| if (!typeInfo.getCategory().equals(Category.LIST)) { |
| throw new UDFArgumentException("Expected list type: " + typeInfo); |
| } |
| return (ListTypeInfo) typeInfo; |
| } |
| |
| public static boolean isSameCategoryGroup(@Nonnull final PrimitiveCategory cat1, |
| @Nonnull final PrimitiveCategory cat2) { |
| if (cat1 == cat2) { |
| return true; |
| } |
| |
| switch (cat1) { |
| // integers |
| case BYTE: |
| case SHORT: |
| case INT: |
| case LONG: { |
| switch (cat2) { |
| case BYTE: |
| case SHORT: |
| case INT: |
| case LONG: |
| return true; |
| default: |
| return false; |
| } |
| } |
| // floating point number |
| case FLOAT: |
| case DOUBLE: { |
| switch (cat2) { |
| case FLOAT: |
| case DOUBLE: |
| return true; |
| default: |
| return false; |
| } |
| } |
| // string |
| case STRING: |
| case CHAR: |
| case VARCHAR: |
| switch (cat2) { |
| case STRING: |
| case CHAR: |
| case VARCHAR: |
| return true; |
| default: |
| return false; |
| } |
| default: |
| break; |
| } |
| return false; |
| } |
| |
| @Nullable |
| public static ArrayList<Object> copyListObject(@Nonnull final DeferredObject argument, |
| @Nonnull final ListObjectInspector loi) throws HiveException { |
| return copyListObject(argument, loi, ObjectInspectorCopyOption.DEFAULT); |
| } |
| |
| @Nullable |
| public static ArrayList<Object> copyListObject(@Nonnull final DeferredObject argument, |
| @Nonnull final ListObjectInspector loi, |
| @Nonnull final ObjectInspectorCopyOption objectInspectorOption) throws HiveException { |
| final Object o = argument.get(); |
| if (o == null) { |
| return null; |
| } |
| |
| final int length = loi.getListLength(o); |
| final ArrayList<Object> list = new ArrayList<Object>(length); |
| for (int i = 0; i < length; i++) { |
| Object e = ObjectInspectorUtils.copyToStandardObject(loi.getListElement(o, i), |
| loi.getListElementObjectInspector(), objectInspectorOption); |
| list.add(e); |
| } |
| return list; |
| } |
| |
| public static float getFloat(@Nullable Object o, @Nonnull PrimitiveObjectInspector oi) { |
| if (o == null) { |
| return 0.f; |
| } |
| return PrimitiveObjectInspectorUtils.getFloat(o, oi); |
| } |
| |
| public static double getDouble(@Nullable Object o, @Nonnull PrimitiveObjectInspector oi) { |
| if (o == null) { |
| return 0.d; |
| } |
| return PrimitiveObjectInspectorUtils.getDouble(o, oi); |
| } |
| |
| public static int getInt(@Nullable Object o, @Nonnull PrimitiveObjectInspector oi) { |
| if (o == null) { |
| return 0; |
| } |
| return PrimitiveObjectInspectorUtils.getInt(o, oi); |
| } |
| |
| public static long getLong(@Nullable Object o, @Nonnull PrimitiveObjectInspector oi) { |
| if (o == null) { |
| return 0L; |
| } |
| return PrimitiveObjectInspectorUtils.getLong(o, oi); |
| } |
| |
| @SuppressWarnings("unchecked") |
| @Nullable |
| public static <T extends Writable> T getConstValue(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| if (!ObjectInspectorUtils.isConstantObjectInspector(oi)) { |
| throw new UDFArgumentException("argument must be a constant value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| ConstantObjectInspector constOI = (ConstantObjectInspector) oi; |
| Object v = constOI.getWritableConstantValue(); |
| return (T) v; |
| } |
| |
| @Nullable |
| public static String[] getConstStringArray(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| if (!ObjectInspectorUtils.isConstantObjectInspector(oi)) { |
| throw new UDFArgumentException("argument must be a constant value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| ConstantObjectInspector constOI = (ConstantObjectInspector) oi; |
| if (constOI.getCategory() != Category.LIST) { |
| throw new UDFArgumentException( |
| "argument must be an array: " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| final List<?> lst = (List<?>) constOI.getWritableConstantValue(); |
| if (lst == null) { |
| return null; |
| } |
| final int size = lst.size(); |
| final String[] ary = new String[size]; |
| for (int i = 0; i < size; i++) { |
| Object o = lst.get(i); |
| if (o != null) { |
| ary[i] = o.toString(); |
| } |
| } |
| return ary; |
| } |
| |
| @Nullable |
| public static double[] getConstDoubleArray(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| if (!ObjectInspectorUtils.isConstantObjectInspector(oi)) { |
| throw new UDFArgumentException("argument must be a constant value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| ConstantObjectInspector constOI = (ConstantObjectInspector) oi; |
| if (constOI.getCategory() != Category.LIST) { |
| throw new UDFArgumentException( |
| "argument must be an array: " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| StandardConstantListObjectInspector listOI = (StandardConstantListObjectInspector) constOI; |
| PrimitiveObjectInspector elemOI = |
| HiveUtils.asDoubleCompatibleOI(listOI.getListElementObjectInspector()); |
| |
| final List<?> lst = listOI.getWritableConstantValue(); |
| if (lst == null) { |
| return null; |
| } |
| final int size = lst.size(); |
| final double[] ary = new double[size]; |
| for (int i = 0; i < size; i++) { |
| Object o = lst.get(i); |
| if (o == null) { |
| ary[i] = Double.NaN; |
| } else { |
| ary[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI); |
| } |
| } |
| return ary; |
| } |
| |
| @Nullable |
| public static String getConstString(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| if (!isStringOI(oi)) { |
| throw new UDFArgumentException("argument must be a Text value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| Text v = getConstValue(oi); |
| return v == null ? null : v.toString(); |
| } |
| |
| |
| @Nullable |
| public static String getConstString(@Nonnull final ObjectInspector[] argOIs, final int argIndex) |
| throws UDFArgumentException { |
| final ObjectInspector oi = getObjectInspector(argOIs, argIndex); |
| if (!isStringOI(oi)) { |
| throw new UDFArgumentTypeException(argIndex, "argument must be a Text value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| Text v = getConstValue(oi); |
| return v == null ? null : v.toString(); |
| } |
| |
| public static boolean getConstBoolean(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| if (!isBooleanOI(oi)) { |
| throw new UDFArgumentException("argument must be a Boolean value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| BooleanWritable v = getConstValue(oi); |
| return v.get(); |
| } |
| |
| |
| public static boolean getConstBoolean(@Nonnull final ObjectInspector[] argOIs, |
| final int argIndex) throws UDFArgumentException { |
| final ObjectInspector oi = getObjectInspector(argOIs, argIndex); |
| if (!isBooleanOI(oi)) { |
| throw new UDFArgumentTypeException(argIndex, "argument must be a Boolean value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| BooleanWritable v = getConstValue(oi); |
| return v.get(); |
| } |
| |
| public static int getConstInt(@Nonnull final ObjectInspector oi) throws UDFArgumentException { |
| if (!isIntOI(oi)) { |
| throw new UDFArgumentException("argument must be a Int value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| IntWritable v = getConstValue(oi); |
| return v.get(); |
| } |
| |
| public static long getConstLong(@Nonnull final ObjectInspector oi) throws UDFArgumentException { |
| if (!isBigIntOI(oi)) { |
| throw new UDFArgumentException("argument must be a BigInt value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| LongWritable v = getConstValue(oi); |
| return v.get(); |
| } |
| |
| public static int getAsConstInt(@Nonnull final ObjectInspector numberOI) |
| throws UDFArgumentException { |
| final String typeName = numberOI.getTypeName(); |
| if (INT_TYPE_NAME.equals(typeName)) { |
| IntWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (BIGINT_TYPE_NAME.equals(typeName)) { |
| LongWritable v = getConstValue(numberOI); |
| return (int) v.get(); |
| } else if (SMALLINT_TYPE_NAME.equals(typeName)) { |
| ShortWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (TINYINT_TYPE_NAME.equals(typeName)) { |
| ByteWritable v = getConstValue(numberOI); |
| return v.get(); |
| } |
| throw new UDFArgumentException("Unexpected argument type to cast as INT: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(numberOI)); |
| } |
| |
| public static long getAsConstLong(@Nonnull final ObjectInspector numberOI) |
| throws UDFArgumentException { |
| final String typeName = numberOI.getTypeName(); |
| if (BIGINT_TYPE_NAME.equals(typeName)) { |
| LongWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (INT_TYPE_NAME.equals(typeName)) { |
| IntWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (SMALLINT_TYPE_NAME.equals(typeName)) { |
| ShortWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (TINYINT_TYPE_NAME.equals(typeName)) { |
| ByteWritable v = getConstValue(numberOI); |
| return v.get(); |
| } |
| throw new UDFArgumentException("Unexpected argument type to cast as long: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(numberOI)); |
| } |
| |
| public static float getAsConstFloat(@Nonnull final ObjectInspector numberOI) |
| throws UDFArgumentException { |
| final String typeName = numberOI.getTypeName(); |
| if (FLOAT_TYPE_NAME.equals(typeName)) { |
| FloatWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (DOUBLE_TYPE_NAME.equals(typeName)) { |
| DoubleWritable v = getConstValue(numberOI); |
| return (float) v.get(); |
| } else if (INT_TYPE_NAME.equals(typeName)) { |
| IntWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (BIGINT_TYPE_NAME.equals(typeName)) { |
| LongWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (SMALLINT_TYPE_NAME.equals(typeName)) { |
| ShortWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (TINYINT_TYPE_NAME.equals(typeName)) { |
| ByteWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (DECIMAL_TYPE_NAME.equals(typeName)) { |
| HiveDecimalWritable v = getConstValue(numberOI); |
| return v.getHiveDecimal().floatValue(); |
| } |
| throw new UDFArgumentException("Unexpected argument type to cast as double: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(numberOI)); |
| } |
| |
| public static double getAsConstDouble(@Nonnull final ObjectInspector numberOI) |
| throws UDFArgumentException { |
| final String typeName = numberOI.getTypeName(); |
| if (DOUBLE_TYPE_NAME.equals(typeName)) { |
| DoubleWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (FLOAT_TYPE_NAME.equals(typeName)) { |
| FloatWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (INT_TYPE_NAME.equals(typeName)) { |
| IntWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (BIGINT_TYPE_NAME.equals(typeName)) { |
| LongWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (SMALLINT_TYPE_NAME.equals(typeName)) { |
| ShortWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (TINYINT_TYPE_NAME.equals(typeName)) { |
| ByteWritable v = getConstValue(numberOI); |
| return v.get(); |
| } else if (DECIMAL_TYPE_NAME.equals(typeName)) { |
| HiveDecimalWritable v = getConstValue(numberOI); |
| return v.getHiveDecimal().doubleValue(); |
| } |
| throw new UDFArgumentException("Unexpected argument type to cast as double: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(numberOI)); |
| } |
| |
| @Nonnull |
| public static long[] asLongArray(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, @Nonnull PrimitiveObjectInspector elemOI) { |
| if (argObj == null) { |
| return null; |
| } |
| final int length = listOI.getListLength(argObj); |
| final long[] ary = new long[length]; |
| for (int i = 0; i < length; i++) { |
| Object o = listOI.getListElement(argObj, i); |
| if (o == null) { |
| continue; |
| } |
| ary[i] = PrimitiveObjectInspectorUtils.getLong(o, elemOI); |
| } |
| return ary; |
| } |
| |
| @Nonnull |
| public static long[] asLongArray(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, @Nonnull LongObjectInspector elemOI) { |
| if (argObj == null) { |
| return null; |
| } |
| final int length = listOI.getListLength(argObj); |
| final long[] ary = new long[length]; |
| for (int i = 0; i < length; i++) { |
| Object o = listOI.getListElement(argObj, i); |
| if (o == null) { |
| continue; |
| } |
| ary[i] = elemOI.get(o); |
| } |
| return ary; |
| } |
| |
| @Nullable |
| public static float[] asFloatArray(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, |
| @Nonnull final PrimitiveObjectInspector elemOI) throws UDFArgumentException { |
| return asFloatArray(argObj, listOI, elemOI, true); |
| } |
| |
| @Nullable |
| public static float[] asFloatArray(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, |
| @Nonnull final PrimitiveObjectInspector elemOI, final boolean avoidNull) |
| throws UDFArgumentException { |
| if (argObj == null) { |
| return null; |
| } |
| final int length = listOI.getListLength(argObj); |
| final float[] ary = new float[length]; |
| for (int i = 0; i < length; i++) { |
| Object o = listOI.getListElement(argObj, i); |
| if (o == null) { |
| if (avoidNull) { |
| continue; |
| } |
| throw new UDFArgumentException("Found null at index " + i); |
| } |
| ary[i] = PrimitiveObjectInspectorUtils.getFloat(o, elemOI); |
| } |
| return ary; |
| } |
| |
| @Nullable |
| public static double[] asDoubleArray(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, |
| @Nonnull final PrimitiveObjectInspector elemOI) throws UDFArgumentException { |
| return asDoubleArray(argObj, listOI, elemOI, true); |
| } |
| |
| @Nullable |
| public static double[] asDoubleArray(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, |
| @Nonnull final PrimitiveObjectInspector elemOI, final boolean avoidNull) |
| throws UDFArgumentException { |
| if (argObj == null) { |
| return null; |
| } |
| final int length = listOI.getListLength(argObj); |
| final double[] ary = new double[length]; |
| for (int i = 0; i < length; i++) { |
| Object o = listOI.getListElement(argObj, i); |
| if (o == null) { |
| if (avoidNull) { |
| continue; |
| } |
| throw new UDFArgumentException("Found null at index " + i); |
| } |
| ary[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI); |
| } |
| return ary; |
| } |
| |
| @Nonnull |
| public static void toDoubleArray(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, |
| @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out, |
| final boolean avoidNull) throws UDFArgumentException { |
| if (argObj == null) { |
| return; |
| } |
| final int length = listOI.getListLength(argObj); |
| if (out.length != length) { |
| throw new UDFArgumentException( |
| "Dimension mismatched. Expected: " + out.length + ", Actual: " + length); |
| } |
| for (int i = 0; i < length; i++) { |
| Object o = listOI.getListElement(argObj, i); |
| if (o == null) { |
| if (avoidNull) { |
| continue; |
| } |
| throw new UDFArgumentException("Found null at index " + i); |
| } |
| out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI); |
| } |
| return; |
| } |
| |
| @Nonnull |
| public static void toDoubleArray(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, |
| @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final double[] out, |
| final double nullValue) throws UDFArgumentException { |
| if (argObj == null) { |
| return; |
| } |
| final int length = listOI.getListLength(argObj); |
| if (out.length != length) { |
| throw new UDFArgumentException( |
| "Dimension mismatched. Expected: " + out.length + ", Actual: " + length); |
| } |
| for (int i = 0; i < length; i++) { |
| Object o = listOI.getListElement(argObj, i); |
| if (o == null) { |
| out[i] = nullValue; |
| continue; |
| } |
| out[i] = PrimitiveObjectInspectorUtils.getDouble(o, elemOI); |
| } |
| return; |
| } |
| |
| /** |
| * @return the number of true bits |
| */ |
| @Nonnull |
| public static int setBits(@Nullable final Object argObj, |
| @Nonnull final ListObjectInspector listOI, |
| @Nonnull final PrimitiveObjectInspector elemOI, @Nonnull final BitSet bitset) |
| throws UDFArgumentException { |
| if (argObj == null) { |
| return 0; |
| } |
| int count = 0; |
| final int length = listOI.getListLength(argObj); |
| for (int i = 0; i < length; i++) { |
| final Object o = listOI.getListElement(argObj, i); |
| if (o == null) { |
| continue; |
| } |
| final int index = PrimitiveObjectInspectorUtils.getInt(o, elemOI); |
| if (index < 0) { |
| throw new UDFArgumentException("Negative index is not allowed: " + index); |
| } |
| bitset.set(index); |
| count++; |
| } |
| return count; |
| } |
| |
| @Nonnull |
| public static ConstantObjectInspector asConstantObjectInspector( |
| @Nonnull final ObjectInspector oi) throws UDFArgumentException { |
| if (!ObjectInspectorUtils.isConstantObjectInspector(oi)) { |
| throw new UDFArgumentException("argument must be a constant value: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| return (ConstantObjectInspector) oi; |
| } |
| |
| public static ObjectInspector getObjectInspector(@Nonnull final ObjectInspector[] argOIs, |
| final int argIndex) throws UDFArgumentException { |
| if (argIndex >= argOIs.length) { |
| throw new UDFArgumentException("Illegal argument index:" + argIndex); |
| } |
| return argOIs[argIndex]; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asPrimitiveObjectInspector( |
| @Nonnull final ObjectInspector oi) throws UDFArgumentException { |
| if (oi.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentException("Expecting PrimitiveObjectInspector: " |
| + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| return (PrimitiveObjectInspector) oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asPrimitiveObjectInspector( |
| @Nonnull final ObjectInspector[] argOIs, final int argIndex) |
| throws UDFArgumentException { |
| final ObjectInspector oi = getObjectInspector(argOIs, argIndex); |
| if (oi.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentException("Expecting PrimitiveObjectInspector for argOIs[" |
| + argIndex + "] but got " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| return (PrimitiveObjectInspector) oi; |
| } |
| |
| @Nonnull |
| public static StringObjectInspector asStringOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentException { |
| if (!isStringOI(argOI)) { |
| throw new UDFArgumentException("Argument type must be String: " + argOI.getTypeName()); |
| } |
| return (StringObjectInspector) argOI; |
| } |
| |
| @Nonnull |
| public static StringObjectInspector asStringOI(@Nonnull final ObjectInspector[] argOIs, |
| final int argIndex) throws UDFArgumentException { |
| final ObjectInspector oi = getObjectInspector(argOIs, argIndex); |
| if (!isStringOI(oi)) { |
| throw new UDFArgumentException( |
| "argOIs[" + argIndex + "] type must be String: " + oi.getTypeName()); |
| } |
| return (StringObjectInspector) oi; |
| } |
| |
| @Nonnull |
| public static BinaryObjectInspector asBinaryOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentException { |
| if (!BINARY_TYPE_NAME.equals(argOI.getTypeName())) { |
| throw new UDFArgumentException("Argument type must be Binary: " + argOI.getTypeName()); |
| } |
| return (BinaryObjectInspector) argOI; |
| } |
| |
| @Nonnull |
| public static BooleanObjectInspector asBooleanOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentException { |
| if (!BOOLEAN_TYPE_NAME.equals(argOI.getTypeName())) { |
| throw new UDFArgumentException("Argument type must be Boolean: " + argOI.getTypeName()); |
| } |
| return (BooleanObjectInspector) argOI; |
| } |
| |
| @Nonnull |
| public static BooleanObjectInspector asBooleanOI(@Nonnull final ObjectInspector[] argOIs, |
| final int argIndex) throws UDFArgumentException { |
| ObjectInspector argOI = getObjectInspector(argOIs, argIndex); |
| if (!BOOLEAN_TYPE_NAME.equals(argOI.getTypeName())) { |
| throw new UDFArgumentTypeException(argIndex, |
| "Argument type must be Boolean: " + argOI.getTypeName()); |
| } |
| return (BooleanObjectInspector) argOI; |
| } |
| |
| @Nonnull |
| public static IntObjectInspector asIntOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentException { |
| if (!INT_TYPE_NAME.equals(argOI.getTypeName())) { |
| throw new UDFArgumentException("Argument type must be INT: " + argOI.getTypeName()); |
| } |
| return (IntObjectInspector) argOI; |
| } |
| |
| @Nonnull |
| public static LongObjectInspector asLongOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentException { |
| if (!BIGINT_TYPE_NAME.equals(argOI.getTypeName())) { |
| throw new UDFArgumentException("Argument type must be BIGINT: " + argOI.getTypeName()); |
| } |
| return (LongObjectInspector) argOI; |
| } |
| |
| @Nonnull |
| public static DoubleObjectInspector asDoubleOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentException { |
| if (!DOUBLE_TYPE_NAME.equals(argOI.getTypeName())) { |
| throw new UDFArgumentException("Argument type must be DOUBLE: " + argOI.getTypeName()); |
| } |
| return (DoubleObjectInspector) argOI; |
| } |
| |
| @Nonnull |
| public static DoubleObjectInspector asDoubleOI(@Nonnull final ObjectInspector[] argOIs, |
| final int argIndex) throws UDFArgumentException { |
| ObjectInspector argOI = getObjectInspector(argOIs, argIndex); |
| if (!DOUBLE_TYPE_NAME.equals(argOI.getTypeName())) { |
| throw new UDFArgumentTypeException(argIndex, |
| "Argument type must be DOUBLE: " + argOI.getTypeName()); |
| } |
| return (DoubleObjectInspector) argOI; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asIntCompatibleOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentTypeException { |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " |
| + argOI.getTypeName() + " is passed."); |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case INT: |
| case SHORT: |
| case LONG: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| case BOOLEAN: |
| case BYTE: |
| case STRING: |
| break; |
| default: |
| throw new UDFArgumentTypeException(0, |
| "Unexpected type '" + argOI.getTypeName() + "' is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asIntCompatibleOI( |
| @Nonnull final ObjectInspector[] argOIs, final int argIndex) |
| throws UDFArgumentException { |
| ObjectInspector argOI = getObjectInspector(argOIs, argIndex); |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentTypeException(argIndex, |
| "Only primitive type arguments are accepted but " + argOI.getTypeName() |
| + " is passed."); |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case INT: |
| case SHORT: |
| case LONG: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| case BOOLEAN: |
| case BYTE: |
| case STRING: |
| break; |
| default: |
| throw new UDFArgumentTypeException(argIndex, |
| "Unexpected type '" + argOI.getTypeName() + "' is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asLongCompatibleOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentTypeException { |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " |
| + argOI.getTypeName() + " is passed."); |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case LONG: |
| case INT: |
| case SHORT: |
| case BYTE: |
| case BOOLEAN: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| case STRING: |
| case TIMESTAMP: |
| break; |
| default: |
| throw new UDFArgumentTypeException(0, |
| "Unexpected type '" + argOI.getTypeName() + "' is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asIntegerOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentTypeException { |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " |
| + argOI.getTypeName() + " is passed."); |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case INT: |
| case SHORT: |
| case LONG: |
| case BYTE: |
| break; |
| default: |
| throw new UDFArgumentTypeException(0, |
| "Unexpected type '" + argOI.getTypeName() + "' is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asIntegerOI(@Nonnull final ObjectInspector[] argOIs, |
| final int argIndex) throws UDFArgumentException { |
| final ObjectInspector argOI = getObjectInspector(argOIs, argIndex); |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentTypeException(argIndex, |
| "Only primitive type arguments are accepted but " + argOI.getTypeName() |
| + " is passed."); |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case INT: |
| case SHORT: |
| case LONG: |
| case BYTE: |
| break; |
| default: |
| throw new UDFArgumentTypeException(argIndex, |
| "Unexpected type '" + argOI.getTypeName() + "' is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asDoubleCompatibleOI( |
| @Nonnull final ObjectInspector argOI) throws UDFArgumentTypeException { |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " |
| + argOI.getTypeName() + " is passed."); |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case BYTE: |
| case SHORT: |
| case INT: |
| case LONG: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| case STRING: |
| case TIMESTAMP: |
| break; |
| default: |
| throw new UDFArgumentTypeException(0, |
| "Only numeric or string type arguments are accepted but " + argOI.getTypeName() |
| + " is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asDoubleCompatibleOI( |
| @Nonnull final ObjectInspector[] argOIs, final int argIndex) |
| throws UDFArgumentException { |
| final PrimitiveObjectInspector oi = asPrimitiveObjectInspector(argOIs, argIndex); |
| switch (oi.getPrimitiveCategory()) { |
| case BYTE: |
| case SHORT: |
| case INT: |
| case LONG: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| case STRING: |
| case TIMESTAMP: |
| break; |
| default: |
| throw new UDFArgumentTypeException(argIndex, |
| "Only numeric or string type arguments are accepted but " + oi.getTypeName() |
| + " is passed for argument index " + argIndex); |
| } |
| return oi; |
| |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asFloatingPointOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentTypeException { |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " |
| + argOI.getTypeName() + " is passed."); |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| break; |
| default: |
| throw new UDFArgumentTypeException(0, "Only floating point number is accepted but " |
| + argOI.getTypeName() + " is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asNumberOI(@Nonnull final ObjectInspector[] argOIs, |
| final int argIndex) throws UDFArgumentException { |
| final PrimitiveObjectInspector oi = asPrimitiveObjectInspector(argOIs, argIndex); |
| switch (oi.getPrimitiveCategory()) { |
| case BYTE: |
| case SHORT: |
| case INT: |
| case LONG: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| break; |
| default: |
| throw new UDFArgumentTypeException(argIndex, |
| "Only numeric argument is accepted but " + oi.getTypeName() + " is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static PrimitiveObjectInspector asNumberOI(@Nonnull final ObjectInspector argOI) |
| throws UDFArgumentTypeException { |
| if (argOI.getCategory() != Category.PRIMITIVE) { |
| throw new UDFArgumentTypeException(0, "Only primitive type arguments are accepted but " |
| + argOI.getTypeName() + " is passed."); |
| } |
| final PrimitiveObjectInspector oi = (PrimitiveObjectInspector) argOI; |
| switch (oi.getPrimitiveCategory()) { |
| case BYTE: |
| case SHORT: |
| case INT: |
| case LONG: |
| case FLOAT: |
| case DOUBLE: |
| case DECIMAL: |
| break; |
| default: |
| throw new UDFArgumentTypeException(0, |
| "Only numeric argument is accepted but " + argOI.getTypeName() + " is passed."); |
| } |
| return oi; |
| } |
| |
| @Nonnull |
| public static ListObjectInspector asListOI(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| Category category = oi.getCategory(); |
| if (category != Category.LIST) { |
| throw new UDFArgumentException("Expected List OI but was: " + oi); |
| } |
| return (ListObjectInspector) oi; |
| } |
| |
| @Nonnull |
| public static ListObjectInspector asListOI(@Nonnull final ObjectInspector[] argOIs, |
| final int argIndex) throws UDFArgumentException { |
| final ObjectInspector oi = getObjectInspector(argOIs, argIndex); |
| Category category = oi.getCategory(); |
| if (category != Category.LIST) { |
| throw new UDFArgumentException("Expecting ListObjectInspector for argOIs[" + argIndex |
| + "] but got " + TypeInfoUtils.getTypeInfoFromObjectInspector(oi)); |
| } |
| return (ListObjectInspector) oi; |
| } |
| |
| |
| @Nonnull |
| public static MapObjectInspector asMapOI(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| if (oi.getCategory() != Category.MAP) { |
| throw new UDFArgumentException("Expected Map OI but was: " + oi); |
| } |
| return (MapObjectInspector) oi; |
| } |
| |
| public static void validateFeatureOI(@Nonnull final ObjectInspector oi) |
| throws UDFArgumentException { |
| final String typeName = oi.getTypeName(); |
| if (!STRING_TYPE_NAME.equals(typeName) && !INT_TYPE_NAME.equals(typeName) |
| && !BIGINT_TYPE_NAME.equals(typeName)) { |
| throw new UDFArgumentException( |
| "argument type for a feature must be List of key type [Int|BitInt|Text]: " |
| + typeName); |
| } |
| } |
| |
| @Nonnull |
| public static FloatWritable[] newFloatArray(final int size, final float defaultVal) { |
| final FloatWritable[] array = new FloatWritable[size]; |
| for (int i = 0; i < size; i++) { |
| array[i] = new FloatWritable(defaultVal); |
| } |
| return array; |
| } |
| |
| public static LazySimpleSerDe getKeyValueLineSerde( |
| @Nonnull final PrimitiveObjectInspector keyOI, |
| @Nonnull final PrimitiveObjectInspector valueOI) throws SerDeException { |
| LazySimpleSerDe serde = new LazySimpleSerDe(); |
| Configuration conf = new Configuration(); |
| Properties tbl = new Properties(); |
| tbl.setProperty("columns", "key,value"); |
| tbl.setProperty("columns.types", keyOI.getTypeName() + "," + valueOI.getTypeName()); |
| serde.initialize(conf, tbl); |
| return serde; |
| } |
| |
| public static LazySimpleSerDe getLineSerde(@Nonnull final PrimitiveObjectInspector... OIs) |
| throws SerDeException { |
| if (OIs.length == 0) { |
| throw new IllegalArgumentException("OIs must be specified"); |
| } |
| LazySimpleSerDe serde = new LazySimpleSerDe(); |
| Configuration conf = new Configuration(); |
| Properties tbl = new Properties(); |
| |
| StringBuilder columnNames = new StringBuilder(); |
| StringBuilder columnTypes = new StringBuilder(); |
| for (int i = 0; i < OIs.length; i++) { |
| columnNames.append('c').append(i + 1).append(','); |
| columnTypes.append(OIs[i].getTypeName()).append(','); |
| } |
| columnNames.deleteCharAt(columnNames.length() - 1); |
| columnTypes.deleteCharAt(columnTypes.length() - 1); |
| |
| tbl.setProperty("columns", columnNames.toString()); |
| tbl.setProperty("columns.types", columnTypes.toString()); |
| serde.initialize(conf, tbl); |
| return serde; |
| } |
| |
| @Nonnull |
| public static Object castLazyBinaryObject(@Nonnull final Object obj) { |
| if (obj instanceof LazyBinaryMap) { |
| return ((LazyBinaryMap) obj).getMap(); |
| } else if (obj instanceof LazyBinaryArray) { |
| return ((LazyBinaryArray) obj).getList(); |
| } |
| return obj; |
| } |
| |
| @Nonnull |
| public static LazyString lazyString(@Nonnull final String str) { |
| return lazyString(str, (byte) '\\'); |
| } |
| |
| @Nonnull |
| public static LazyString lazyString(@Nonnull final String str, final byte escapeChar) { |
| LazyStringObjectInspector oi = |
| LazyPrimitiveObjectInspectorFactory.getLazyStringObjectInspector(false, escapeChar); |
| return lazyString(str, oi); |
| } |
| |
| @Nonnull |
| public static LazyString lazyString(@Nonnull final String str, |
| @Nonnull final LazyStringObjectInspector oi) { |
| LazyString lazy = new LazyString(oi); |
| ByteArrayRef ref = new ByteArrayRef(); |
| byte[] data = str.getBytes(StandardCharsets.UTF_8); |
| ref.setData(data); |
| lazy.init(ref, 0, data.length); |
| return lazy; |
| } |
| |
| @Nonnull |
| public static LazyInteger lazyInteger(@Nonnull final int v) { |
| LazyInteger lazy = |
| new LazyInteger(LazyPrimitiveObjectInspectorFactory.LAZY_INT_OBJECT_INSPECTOR); |
| lazy.getWritableObject().set(v); |
| return lazy; |
| } |
| |
| @Nonnull |
| public static LazyLong lazyLong(@Nonnull final long v) { |
| LazyLong lazy = |
| new LazyLong(LazyPrimitiveObjectInspectorFactory.LAZY_LONG_OBJECT_INSPECTOR); |
| lazy.getWritableObject().set(v); |
| return lazy; |
| } |
| |
| @Nonnull |
| public static ObjectInspector getObjectInspector(@Nonnull final String typeString, |
| final boolean preferWritable) { |
| TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(typeString); |
| if (preferWritable) { |
| return TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo); |
| } else { |
| return TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo); |
| } |
| } |
| |
| @Nonnull |
| public static WritableConstantStringObjectInspector getConstStringObjectInspector( |
| @Nonnull final String str) { |
| return (WritableConstantStringObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector( |
| TypeInfoFactory.stringTypeInfo, new Text(str)); |
| } |
| |
| @Nullable |
| public static StructField getStructFieldRef(@Nonnull String fieldName, |
| @Nonnull final List<? extends StructField> fields) { |
| fieldName = fieldName.toLowerCase(); |
| for (StructField f : fields) { |
| if (f.getFieldName().equals(fieldName)) { |
| return f; |
| } |
| } |
| // For backward compatibility: fieldNames can also be integer Strings. |
| try { |
| final int i = Integer.parseInt(fieldName); |
| if (i >= 0 && i < fields.size()) { |
| return fields.get(i); |
| } |
| } catch (NumberFormatException e) { |
| // ignore |
| } |
| return null; |
| } |
| } |
