| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package edu.uci.ics.hivesterix.test.serde; |
| |
| import java.util.List; |
| import java.util.Properties; |
| |
| import junit.framework.TestCase; |
| |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.hive.serde.Constants; |
| import org.apache.hadoop.hive.serde2.SerDe; |
| import org.apache.hadoop.hive.serde2.SerDeException; |
| import org.apache.hadoop.hive.serde2.io.ByteWritable; |
| import org.apache.hadoop.hive.serde2.io.DoubleWritable; |
| import org.apache.hadoop.hive.serde2.io.ShortWritable; |
| import org.apache.hadoop.hive.serde2.lazy.LazyPrimitive; |
| import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; |
| import org.apache.hadoop.hive.serde2.objectinspector.StructField; |
| import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; |
| import org.apache.hadoop.io.BytesWritable; |
| import org.apache.hadoop.io.IntWritable; |
| import org.apache.hadoop.io.LongWritable; |
| import org.apache.hadoop.io.Text; |
| import org.apache.hadoop.io.Writable; |
| |
| import edu.uci.ics.hivesterix.serde.lazy.LazySerDe; |
| |
| /** |
| * TestLazySimpleSerDe. |
| * |
| */ |
| @SuppressWarnings({ "deprecation", "rawtypes" }) |
| public class SerDeTest extends TestCase { |
| |
| /** |
| * Test the LazySimpleSerDe class. |
| */ |
| public void testLazySimpleSerDe() throws Throwable { |
| try { |
| // Create the SerDe |
| LazySimpleSerDe serDe = new LazySimpleSerDe(); |
| Configuration conf = new Configuration(); |
| Properties tbl = createProperties(); |
| serDe.initialize(conf, tbl); |
| |
| LazySerDe outputSerde = new LazySerDe(); |
| outputSerde.initialize(conf, tbl); |
| |
| // Data |
| String s = "123\t456\t789\t1000\t5.3\thive and hadoop\t1\tqf"; |
| |
| byte[] bytes = s.getBytes(); |
| Writable bytesWritable = new BytesWritable(bytes); |
| |
| // Test |
| // deserializeAndSerialize(serDe, t, s, expectedFieldsData); |
| Object row = serDe.deserialize(bytesWritable); // test my serde |
| StructObjectInspector simpleInspector = (StructObjectInspector) serDe |
| .getObjectInspector(); |
| List<Object> fields = simpleInspector |
| .getStructFieldsDataAsList(row); |
| List<? extends StructField> fieldRefs = simpleInspector |
| .getAllStructFieldRefs(); |
| |
| int i = 0; |
| for (Object field : fields) { |
| BytesWritable fieldWritable = (BytesWritable) outputSerde |
| .serialize(field, fieldRefs.get(i) |
| .getFieldObjectInspector()); |
| System.out.print(fieldWritable.getSize() + "|"); |
| i++; |
| } |
| |
| // Writable output = outputSerde.serialize(row, serDe |
| // .getObjectInspector()); |
| // System.out.println(output); |
| // |
| // Object row2 = outputSerde.deserialize(output); |
| // Writable output2 = serDe.serialize(row2, outputSerde |
| // .getObjectInspector()); |
| // System.out.println(output2); |
| |
| // System.out.println(output); |
| // deserializeAndSerialize(outputSerde, t, s, expectedFieldsData); |
| |
| } catch (Throwable e) { |
| e.printStackTrace(); |
| throw e; |
| } |
| } |
| |
| private void deserializeAndSerialize(SerDe serDe, Text t, String s, |
| Object[] expectedFieldsData) throws SerDeException { |
| // Get the row structure |
| StructObjectInspector oi = (StructObjectInspector) serDe |
| .getObjectInspector(); |
| List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs(); |
| assertEquals(8, fieldRefs.size()); |
| |
| // Deserialize |
| Object row = serDe.deserialize(t); |
| for (int i = 0; i < fieldRefs.size(); i++) { |
| Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i)); |
| if (fieldData != null) { |
| fieldData = ((LazyPrimitive) fieldData).getWritableObject(); |
| } |
| assertEquals("Field " + i, expectedFieldsData[i], fieldData); |
| } |
| // Serialize |
| assertEquals(Text.class, serDe.getSerializedClass()); |
| Text serializedText = (Text) serDe.serialize(row, oi); |
| assertEquals("Serialized data", s, serializedText.toString()); |
| } |
| |
| private Properties createProperties() { |
| Properties tbl = new Properties(); |
| |
| // Set the configuration parameters |
| tbl.setProperty(Constants.SERIALIZATION_FORMAT, "9"); |
| tbl.setProperty("columns", |
| "abyte,ashort,aint,along,adouble,astring,anullint,anullstring"); |
| tbl.setProperty("columns.types", |
| "tinyint:smallint:int:bigint:double:string:int:string"); |
| tbl.setProperty(Constants.SERIALIZATION_NULL_FORMAT, "NULL"); |
| return tbl; |
| } |
| |
| /** |
| * Test the LazySimpleSerDe class with LastColumnTakesRest option. |
| */ |
| public void testLazySimpleSerDeLastColumnTakesRest() throws Throwable { |
| try { |
| // Create the SerDe |
| LazySimpleSerDe serDe = new LazySimpleSerDe(); |
| Configuration conf = new Configuration(); |
| Properties tbl = createProperties(); |
| tbl.setProperty(Constants.SERIALIZATION_LAST_COLUMN_TAKES_REST, |
| "true"); |
| serDe.initialize(conf, tbl); |
| |
| // Data |
| Text t = new Text( |
| "123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t"); |
| String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta\tb\t"; |
| Object[] expectedFieldsData = { new ByteWritable((byte) 123), |
| new ShortWritable((short) 456), new IntWritable(789), |
| new LongWritable(1000), new DoubleWritable(5.3), |
| new Text("hive and hadoop"), null, new Text("a\tb\t") }; |
| |
| // Test |
| deserializeAndSerialize(serDe, t, s, expectedFieldsData); |
| |
| } catch (Throwable e) { |
| e.printStackTrace(); |
| throw e; |
| } |
| } |
| |
| /** |
| * Test the LazySimpleSerDe class with extra columns. |
| */ |
| public void testLazySimpleSerDeExtraColumns() throws Throwable { |
| try { |
| // Create the SerDe |
| LazySimpleSerDe serDe = new LazySimpleSerDe(); |
| Configuration conf = new Configuration(); |
| Properties tbl = createProperties(); |
| serDe.initialize(conf, tbl); |
| |
| // Data |
| Text t = new Text( |
| "123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t"); |
| String s = "123\t456\t789\t1000\t5.3\thive and hadoop\tNULL\ta"; |
| Object[] expectedFieldsData = { new ByteWritable((byte) 123), |
| new ShortWritable((short) 456), new IntWritable(789), |
| new LongWritable(1000), new DoubleWritable(5.3), |
| new Text("hive and hadoop"), null, new Text("a") }; |
| |
| // Test |
| deserializeAndSerialize(serDe, t, s, expectedFieldsData); |
| |
| } catch (Throwable e) { |
| e.printStackTrace(); |
| throw e; |
| } |
| } |
| |
| /** |
| * Test the LazySimpleSerDe class with missing columns. |
| */ |
| public void testLazySimpleSerDeMissingColumns() throws Throwable { |
| try { |
| // Create the SerDe |
| LazySimpleSerDe serDe = new LazySimpleSerDe(); |
| Configuration conf = new Configuration(); |
| Properties tbl = createProperties(); |
| serDe.initialize(conf, tbl); |
| |
| // Data |
| Text t = new Text("123\t456\t789\t1000\t5.3\t"); |
| String s = "123\t456\t789\t1000\t5.3\t\tNULL\tNULL"; |
| Object[] expectedFieldsData = { new ByteWritable((byte) 123), |
| new ShortWritable((short) 456), new IntWritable(789), |
| new LongWritable(1000), new DoubleWritable(5.3), |
| new Text(""), null, null }; |
| |
| // Test |
| deserializeAndSerialize(serDe, t, s, expectedFieldsData); |
| |
| } catch (Throwable e) { |
| e.printStackTrace(); |
| throw e; |
| } |
| } |
| |
| } |