====
---- QUERY
# Create a table and populate with data file
create table resolution_by_name_test stored as parquet
as select * from functional_parquet.tinytable;
select a, b from resolution_by_name_test;
---- TYPES
string,string
---- RESULTS
'aaaaaaa','bbbbbbb'
'ccccc','dddd'
'eeeeeeee','f'
====
---- QUERY
# Rearrange the columns and make sure we can still resolve by name
alter table resolution_by_name_test replace columns (b string, a string);
set parquet_fallback_schema_resolution="NAME";
select a, b from resolution_by_name_test;
---- TYPES
string,string
---- RESULTS
'aaaaaaa','bbbbbbb'
'ccccc','dddd'
'eeeeeeee','f'
====
---- QUERY
# Renaming a column will cause the column to not be resolved
set parquet_fallback_schema_resolution="NAME";
alter table resolution_by_name_test change a new_a string;
select new_a from resolution_by_name_test;
---- TYPES
string
---- RESULTS
'NULL'
'NULL'
'NULL'
====
---- QUERY
# Can still resolve by ordinal
set parquet_fallback_schema_resolution="POSITION";
select b, new_a from resolution_by_name_test;
---- TYPES
string,string
---- RESULTS
'aaaaaaa','bbbbbbb'
'ccccc','dddd'
'eeeeeeee','f'
====
---- QUERY
# Check that we can parse the integer enum value as well
set parquet_fallback_schema_resolution=1;
select new_a from resolution_by_name_test;
---- TYPES
string
---- RESULTS
'NULL'
'NULL'
'NULL'
====
---- QUERY
set parquet_fallback_schema_resolution=0;
select b, new_a from resolution_by_name_test;
---- TYPES
string,string
---- RESULTS
'aaaaaaa','bbbbbbb'
'ccccc','dddd'
'eeeeeeee','f'
====
---- QUERY
# Test nested types resolution
create table nested_resolution_by_name_test like functional_parquet.complextypestbl;
====
---- SHELL
hdfs dfs -cp -d -f $FILESYSTEM_PREFIX/test-warehouse/complextypestbl_parquet/nullable.parq \
$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/nested_resolution_by_name_test/
hdfs dfs -cp -d -f $FILESYSTEM_PREFIX/test-warehouse/complextypestbl_parquet/nonnullable.parq \
$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/nested_resolution_by_name_test/
====
---- QUERY
select id, nested_struct.a, b.item
from nested_resolution_by_name_test t, t.nested_struct.b
---- TYPES
bigint,int,int
---- RESULTS
1,1,1
2,NULL,NULL
7,7,2
7,7,3
7,7,NULL
8,-1,-1
====
---- QUERY
# Can safely ignore extra fields in nested_struct
alter table nested_resolution_by_name_test change nested_struct nested_struct
struct<a:int, b: array<int>>;
select id, nested_struct.a, b.item
from nested_resolution_by_name_test t, t.nested_struct.b
---- TYPES
bigint,int,int
---- RESULTS
1,1,1
2,NULL,NULL
7,7,2
7,7,3
7,7,NULL
8,-1,-1
====
---- QUERY
# Rearrange nested_struct's fields and make sure we can still resolve by name
alter table nested_resolution_by_name_test change nested_struct nested_struct
struct<b: array<int>, a: int>;
set parquet_fallback_schema_resolution="name";
select id, nested_struct.a, b.item
from nested_resolution_by_name_test t, t.nested_struct.b
---- TYPES
bigint,int,int
---- RESULTS
1,1,1
2,NULL,NULL
7,7,2
7,7,3
7,7,NULL
8,-1,-1
====
---- QUERY
# Can add back a single field
alter table nested_resolution_by_name_test change nested_struct nested_struct
struct<b: array<int>, a: int, g: map<string, struct<h: struct<i: array<float>>>>>;
set parquet_fallback_schema_resolution="name";
select id, g.key
from nested_resolution_by_name_test t, t.nested_struct.g
---- TYPES
bigint,string
---- RESULTS
1,'foo'
2,'g1'
2,'g2'
2,'g3'
2,'g4'
2,'g5'
5,'foo'
====
---- QUERY
# Add back single more nested field (and remove 'g' field)
alter table nested_resolution_by_name_test change nested_struct nested_struct
struct<b: array<int>, a: int, c: struct<d: array<array<struct<f: string>>>>>;
set parquet_fallback_schema_resolution="name";
select tmp.f from nested_resolution_by_name_test.nested_struct.c.d.item tmp;
---- TYPES
string
---- RESULTS
'aaa'
'bbb'
'c'
'NULL'
'aaa'
'NULL'
'bbb'
'NULL'
'c'
'NULL'
'NULL'
'nonnullable'
====
---- QUERY
# Can't rename nested field
alter table nested_resolution_by_name_test change nested_struct nested_struct
struct<b: array<int>, a: int, c: struct<d: array<array<struct<renamed: string>>>>>;
set parquet_fallback_schema_resolution="name";
select tmp.renamed from nested_resolution_by_name_test.nested_struct.c.d.item tmp;
---- TYPES
string
---- RESULTS
'NULL'
'NULL'
'NULL'
'NULL'
'NULL'
'NULL'
'NULL'
'NULL'
'NULL'
'NULL'
'NULL'
'NULL'
====
---- QUERY
# Test switched key/value map fields
create table switched_map_fields_resolution_test (int_map map<string,int>)
stored as parquet;
====
---- SHELL
hdfs dfs -copyFromLocal -d -f \
$IMPALA_HOME/testdata/parquet_schema_resolution/switched_map.parq \
$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/switched_map_fields_resolution_test/
====
---- QUERY
# Switched map fields should be resolvable by name.
set parquet_fallback_schema_resolution="name";
select key, value from switched_map_fields_resolution_test.int_map
---- TYPES
string,int
---- RESULTS
'a',1
'b',2
'c',3
====
---- QUERY
# Can't resolve switched map fields by position since types are switched.
set parquet_fallback_schema_resolution="position";
select key, value from switched_map_fields_resolution_test.int_map
---- CATCH
File '$NAMENODE/test-warehouse/$DATABASE.db/switched_map_fields_resolution_test/
switched_map.parq' has an incompatible Parquet schema for column
 '$DATABASE.switched_map_fields_resolution_test.int_map.key'.
 Column type: STRING, Parquet schema:
required int32 value [i:0 d:1 r:1]
====
---- QUERY
# Check that we handle bad options gracefully
set parquet_fallback_schema_resolution="FOO"
---- CATCH
Invalid parquet fallback schema resolution: 'FOO'. Valid values are POSITION(0), NAME(1).
====