| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.pig.test; |
| |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertNotNull; |
| import static org.junit.Assert.assertTrue; |
| |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.List; |
| |
| import junit.framework.Assert; |
| |
| import org.apache.pig.ResourceSchema; |
| import org.apache.pig.SortColInfo; |
| import org.apache.pig.SortInfo; |
| import org.apache.pig.ResourceSchema.ResourceFieldSchema; |
| import org.apache.pig.backend.executionengine.ExecException; |
| import org.apache.pig.data.DataType; |
| import org.apache.pig.impl.logicalLayer.FrontendException; |
| import org.apache.pig.impl.logicalLayer.schema.Schema; |
| import org.apache.pig.impl.logicalLayer.schema.SchemaMergeException; |
| import org.apache.pig.impl.util.Utils; |
| import org.apache.pig.test.utils.TypeCheckingTestUtil; |
| import org.junit.Test; |
| |
| public class TestResourceSchema { |
| |
| /** |
| * Test that ResourceSchema is correctly created given a |
| * pig.Schema and vice versa |
| */ |
| @Test |
| public void testResourceFlatSchemaCreation() |
| throws ExecException, SchemaMergeException, FrontendException { |
| String [] aliases ={"f1", "f2"}; |
| byte[] types = {DataType.CHARARRAY, DataType.INTEGER}; |
| Schema origSchema = TypeCheckingTestUtil.genFlatSchema( |
| aliases,types); |
| ResourceSchema rsSchema = new ResourceSchema(origSchema); |
| assertEquals("num fields", aliases.length, rsSchema.getFields().length); |
| ResourceSchema.ResourceFieldSchema[] fields = rsSchema.getFields(); |
| for (int i=0; i<fields.length; i++) { |
| assertEquals(fields[i].getName(), aliases[i]); |
| assertEquals(fields[i].getType(), types[i]); |
| } |
| Schema genSchema = Schema.getPigSchema(rsSchema); |
| assertTrue("generated schema equals original", |
| Schema.equals(genSchema, origSchema, true, false)); |
| } |
| |
| /** |
| * Test that ResourceSchema is correctly created given a |
| * pig.Schema and vice versa |
| */ |
| @Test |
| public void testResourceFlatSchemaCreation2() |
| throws ExecException, SchemaMergeException, FrontendException { |
| String [] aliases ={"f1", "f2"}; |
| byte[] types = {DataType.CHARARRAY, DataType.INTEGER}; |
| |
| Schema origSchema = new Schema( |
| new Schema.FieldSchema("t1", |
| new Schema( |
| new Schema.FieldSchema("t0", |
| TypeCheckingTestUtil.genFlatSchema( |
| aliases,types), |
| DataType.TUPLE)), DataType.BAG)); |
| |
| ResourceSchema rsSchema = new ResourceSchema(origSchema); |
| |
| Schema genSchema = Schema.getPigSchema(rsSchema); |
| assertTrue("generated schema equals original", |
| Schema.equals(genSchema, origSchema, true, false)); |
| } |
| |
| /** |
| * Test that ResourceSchema is correctly with SortInfo |
| */ |
| @Test |
| public void testResourceFlatSchemaCreationWithSortInfo() |
| throws ExecException, SchemaMergeException, FrontendException { |
| String [] aliases ={"f1", "f2"}; |
| byte[] types = {DataType.CHARARRAY, DataType.INTEGER}; |
| |
| Schema origSchema = new Schema( |
| new Schema.FieldSchema("t1", |
| new Schema( |
| new Schema.FieldSchema("t0", |
| TypeCheckingTestUtil.genFlatSchema( |
| aliases,types), |
| DataType.TUPLE)), DataType.BAG)); |
| List<SortColInfo> colList = new ArrayList<SortColInfo>(); |
| SortColInfo col1 = new SortColInfo("f1", 0, SortColInfo.Order.ASCENDING); |
| SortColInfo col2 = new SortColInfo("f1", 1, SortColInfo.Order.DESCENDING); |
| colList.add(col1); |
| colList.add(col2); |
| SortInfo sortInfo = new SortInfo(colList); |
| |
| ResourceSchema rsSchema = new ResourceSchema(origSchema, sortInfo); |
| |
| Schema genSchema = Schema.getPigSchema(rsSchema); |
| assertTrue("generated schema equals original", |
| Schema.equals(genSchema, origSchema, true, false)); |
| assertTrue(rsSchema.getSortKeys()[0]==0); |
| assertTrue(rsSchema.getSortKeys()[1]==1); |
| assertTrue(rsSchema.getSortKeyOrders()[0]==ResourceSchema.Order.ASCENDING); |
| assertTrue(rsSchema.getSortKeyOrders()[1]==ResourceSchema.Order.DESCENDING); |
| } |
| |
| /** |
| * Test that Pig Schema is correctly created given a |
| * ResourceSchema and vice versa. Test also that |
| * TwoLevelAccess flag is set for Pig Schema when needed. |
| * @throws IOException |
| */ |
| @Test |
| public void testToPigSchemaWithTwoLevelAccess() throws IOException { |
| ResourceFieldSchema[] level0 = |
| new ResourceFieldSchema[] { |
| new ResourceFieldSchema() |
| .setName("fld0").setType(DataType.CHARARRAY), |
| new ResourceFieldSchema() |
| .setName("fld1").setType(DataType.DOUBLE), |
| new ResourceFieldSchema() |
| .setName("fld2").setType(DataType.INTEGER) |
| }; |
| |
| ResourceSchema rSchema0 = new ResourceSchema() |
| .setFields(level0); |
| |
| ResourceFieldSchema[] level1 = |
| new ResourceFieldSchema[] { |
| new ResourceFieldSchema() |
| .setName("t1").setType(DataType.TUPLE) |
| .setSchema(rSchema0) |
| }; |
| |
| ResourceSchema rSchema1 = new ResourceSchema() |
| .setFields(level1); |
| |
| ResourceFieldSchema[] level2 = |
| new ResourceFieldSchema[] { |
| new ResourceFieldSchema() |
| .setName("t2").setType(DataType.BAG) |
| .setSchema(rSchema1) |
| }; |
| |
| ResourceSchema origSchema = new ResourceSchema() |
| .setFields(level2); |
| |
| Schema pSchema = Schema.getPigSchema(origSchema); |
| |
| assertTrue(!CheckTwoLevelAccess(pSchema)); |
| |
| assertTrue(ResourceSchema.equals(origSchema, new ResourceSchema(pSchema))); |
| } |
| |
| private boolean CheckTwoLevelAccess(Schema s) { |
| if (s == null) return false; |
| for (Schema.FieldSchema fs : s.getFields()) { |
| if (fs.type == DataType.BAG |
| && fs.schema != null |
| && fs.schema.isTwoLevelAccessRequired()) { |
| return true; |
| } |
| if (CheckTwoLevelAccess(fs.schema)) return true; |
| } |
| return false; |
| } |
| |
| /** |
| * Test invalid Resource Schema: multiple fields for a bag |
| * @throws IOException |
| */ |
| @Test(expected=FrontendException.class) |
| public void testToPigSchemaWithInvalidSchema() throws IOException { |
| ResourceFieldSchema[] level0 = new ResourceFieldSchema[] { |
| new ResourceFieldSchema() |
| .setName("fld0").setType(DataType.CHARARRAY), |
| new ResourceFieldSchema() |
| .setName("fld1").setType(DataType.DOUBLE), |
| new ResourceFieldSchema() |
| .setName("fld2").setType(DataType.INTEGER) |
| }; |
| |
| ResourceSchema rSchema0 = new ResourceSchema() |
| .setFields(level0); |
| |
| ResourceFieldSchema[] level2 = new ResourceFieldSchema[] { |
| new ResourceFieldSchema() |
| .setName("t2").setType(DataType.BAG).setSchema(rSchema0) |
| }; |
| } |
| |
| /** |
| * Test invalid Resource Schema: bag without tuple field |
| * @throws IOException |
| */ |
| @Test(expected=FrontendException.class) |
| public void testToPigSchemaWithInvalidSchema2() throws IOException { |
| ResourceFieldSchema[] level0 = new ResourceFieldSchema[] { |
| new ResourceFieldSchema() |
| .setName("fld0").setType(DataType.CHARARRAY) |
| }; |
| |
| ResourceSchema rSchema0 = new ResourceSchema() |
| .setFields(level0); |
| |
| ResourceFieldSchema[] level2 = new ResourceFieldSchema[] { |
| new ResourceFieldSchema() |
| .setName("t2").setType(DataType.BAG).setSchema(rSchema0) |
| }; |
| |
| } |
| |
| /** |
| * Test one-level Pig Schema: multiple fields for a bag |
| */ |
| @Test |
| public void testResourceSchemaWithInvalidPigSchema() |
| throws FrontendException { |
| String [] aliases ={"f1", "f2"}; |
| byte[] types = {DataType.CHARARRAY, DataType.INTEGER}; |
| Schema level0 = TypeCheckingTestUtil.genFlatSchema( |
| aliases,types); |
| Schema.FieldSchema fld0 = |
| new Schema.FieldSchema("f0", level0, DataType.BAG); |
| Schema level1 = new Schema(fld0); |
| try { |
| Schema.getPigSchema(new ResourceSchema(level1)); |
| Assert.fail(); |
| } catch(FrontendException e) { |
| assertTrue(e.getErrorCode()==2218); |
| } |
| } |
| |
| /** |
| * Test one-level Pig Schema: bag without tuple field |
| */ |
| @Test |
| public void testResourceSchemaWithInvalidPigSchema2() |
| throws FrontendException { |
| String [] aliases ={"f1"}; |
| byte[] types = {DataType.INTEGER}; |
| Schema level0 = TypeCheckingTestUtil.genFlatSchema( |
| aliases,types); |
| Schema.FieldSchema fld0 = |
| new Schema.FieldSchema("f0", level0, DataType.BAG); |
| Schema level1 = new Schema(fld0); |
| try { |
| Schema.getPigSchema(new ResourceSchema(level1)); |
| Assert.fail(); |
| } catch (FrontendException e) { |
| assertTrue(e.getErrorCode()==2218); |
| } |
| } |
| |
| /** |
| * Test that we can turn a schema into a string and then parse it |
| * back into a schema. |
| */ |
| @Test |
| public void testToStringAndParse() throws Exception { |
| ResourceSchema rs = new ResourceSchema(); |
| ResourceFieldSchema[] fields = new ResourceFieldSchema[13]; |
| |
| byte[] types = {DataType.INTEGER, DataType.LONG, DataType.FLOAT, |
| DataType.DOUBLE, DataType.BYTEARRAY, DataType.CHARARRAY, |
| DataType.MAP, DataType.TUPLE, DataType.TUPLE, DataType.BAG, |
| DataType.BAG, DataType.BOOLEAN, DataType.DATETIME}; |
| String[] names = {"i", "l", "f", |
| "d", "b", "s", |
| "m", "tschema", "tnull", "bschema", |
| "bnull", "bb", "dt"}; |
| |
| for (int i = 0; i < fields.length; i++) { |
| fields[i] = new ResourceFieldSchema(); |
| fields[i].setName(names[i]); |
| fields[i].setType(types[i]); |
| } |
| |
| // Add in the schemas for the tuple and the bag |
| ResourceSchema tschema = new ResourceSchema(); |
| ResourceFieldSchema[] tfields = new ResourceFieldSchema[3]; |
| for (int i = 0; i < 3; i++) { |
| tfields[i] = new ResourceFieldSchema(); |
| tfields[i].setName(names[i]); |
| tfields[i].setType(types[i]); |
| } |
| tschema.setFields(tfields); |
| fields[7].setSchema(tschema); |
| |
| ResourceSchema bschema = new ResourceSchema(); |
| ResourceFieldSchema[] bfields = new ResourceFieldSchema[1]; |
| bfields[0] = new ResourceFieldSchema(); |
| bfields[0].setName("t"); |
| bfields[0].setType(DataType.TUPLE); |
| |
| ResourceSchema tbschema = new ResourceSchema(); |
| ResourceFieldSchema[] tbfields = new ResourceFieldSchema[3]; |
| for (int i = 3; i < 6; i++) { |
| tbfields[i-3] = new ResourceFieldSchema(); |
| tbfields[i-3].setName(names[i]); |
| tbfields[i-3].setType(types[i]); |
| } |
| tbschema.setFields(tbfields); |
| |
| bfields[0].setSchema(tbschema); |
| bschema.setFields(bfields); |
| |
| fields[9].setSchema(bschema); |
| |
| rs.setFields(fields); |
| |
| String strSchema = rs.toString(); |
| |
| assertEquals("i:int,l:long,f:float,d:double,b:bytearray,s:" + |
| "chararray,m:[],tschema:(i:int,l:long,f:float)," + |
| "tnull:(),bschema:{t:(d:double,b:bytearray,s:chararray)},bnull:{},bb:boolean,dt:datetime", |
| strSchema); |
| |
| ResourceSchema after = |
| new ResourceSchema(Utils.getSchemaFromString(strSchema)); |
| ResourceFieldSchema[] afterFields = after.getFields(); |
| |
| assertEquals(13, afterFields.length); |
| assertEquals("i", afterFields[0].getName()); |
| assertEquals(DataType.INTEGER, afterFields[0].getType()); |
| assertEquals("l", afterFields[1].getName()); |
| assertEquals(DataType.LONG, afterFields[1].getType()); |
| assertEquals("f", afterFields[2].getName()); |
| assertEquals(DataType.FLOAT, afterFields[2].getType()); |
| assertEquals("d", afterFields[3].getName()); |
| assertEquals(DataType.DOUBLE, afterFields[3].getType()); |
| assertEquals("b", afterFields[4].getName()); |
| assertEquals(DataType.BYTEARRAY, afterFields[4].getType()); |
| assertEquals("s", afterFields[5].getName()); |
| assertEquals(DataType.CHARARRAY, afterFields[5].getType()); |
| assertEquals("m", afterFields[6].getName()); |
| assertEquals(DataType.MAP, afterFields[6].getType()); |
| assertEquals("tschema", afterFields[7].getName()); |
| assertEquals(DataType.TUPLE, afterFields[7].getType()); |
| assertEquals("tnull", afterFields[8].getName()); |
| assertEquals(DataType.TUPLE, afterFields[8].getType()); |
| assertEquals("bschema", afterFields[9].getName()); |
| assertEquals(DataType.BAG, afterFields[9].getType()); |
| assertEquals("bnull", afterFields[10].getName()); |
| assertEquals(DataType.BAG, afterFields[10].getType()); |
| assertEquals("bb", afterFields[11].getName()); |
| assertEquals(DataType.BOOLEAN, afterFields[11].getType()); |
| assertEquals("dt", afterFields[12].getName()); |
| assertEquals(DataType.DATETIME, afterFields[12].getType()); |
| |
| |
| assertNotNull(afterFields[7].getSchema()); |
| ResourceFieldSchema[] tAfterFields = |
| afterFields[7].getSchema().getFields(); |
| assertEquals(3, tAfterFields.length); |
| assertEquals("i", tAfterFields[0].getName()); |
| assertEquals(DataType.INTEGER, tAfterFields[0].getType()); |
| assertEquals("l", tAfterFields[1].getName()); |
| assertEquals(DataType.LONG, tAfterFields[1].getType()); |
| assertEquals("f", tAfterFields[2].getName()); |
| assertEquals(DataType.FLOAT, tAfterFields[2].getType()); |
| |
| assertNotNull(afterFields[9].getSchema()); |
| ResourceFieldSchema[] bAfterFields = |
| afterFields[9].getSchema().getFields(); |
| assertEquals(1, bAfterFields.length); |
| assertNotNull(bAfterFields[0].getSchema()); |
| |
| ResourceFieldSchema[] tbAfterFields = |
| bAfterFields[0].getSchema().getFields(); |
| assertEquals(3, tbAfterFields.length); |
| assertEquals("d", tbAfterFields[0].getName()); |
| assertEquals(DataType.DOUBLE, tbAfterFields[0].getType()); |
| assertEquals("b", tbAfterFields[1].getName()); |
| assertEquals(DataType.BYTEARRAY, tbAfterFields[1].getType()); |
| assertEquals("s", tbAfterFields[2].getName()); |
| assertEquals(DataType.CHARARRAY, tbAfterFields[2].getType()); |
| } |
| } |