blob: 81b69086c51ef87ac4e314ee0552969441fb2c1f [file] [log] [blame]
--
-- external tables - short and simple functional tests.
--
-- start_matchsubs
--
-- # replace return code in error message (platform specific)
--
-- m/ERROR\:\s+external table .* command ended with .* not found/
-- s/nosuchcommand\:\s*(command)? not found/nosuchcommand\: NOT FOUND/
--
-- m/ERROR\:\s+external table .* command ended with .*No such file.*/
-- s/nosuchfile\.txt\:\s*No such file (or directory)?/nosuchfile\.txt\: NO SUCH FILE/
-- m/ERROR\:\s+external table .* command ended with .*No such file.*/i
-- s/cat\: (cannot open)? nosuchfile\.txt/cat\: nosuchfile\.txt/
--
-- # remove line number - redhat
-- m/ERROR\:\s+external table .* command ended with .*NOT FOUND.*/i
-- s/\s+line \d+\://
-- # remove line number - Debian
-- m/ERROR\:\s+external table .* command ended with .*sh: 1: .*NOT FOUND.*/i
-- s/ sh: 1: / sh: /
--
-- m/DETAIL: Found \d+ URLs and \d+ primary segments./
-- s/Found.+//
--
-- end_matchsubs
set optimizer_trace_fallback=on;
create schema part_external_table;
set search_path=part_external_table;
create table part (a int, b int) partition by range (b);
NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table.
HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
create external table p1_e (a int, b int) location ('file://@hostname@@abs_srcdir@/data/part1.csv') format 'csv';
create external table p2_e (a int, b int) location ('file://@hostname@@abs_srcdir@/data/part2.csv') format 'csv';
alter table part attach partition p1_e for values from (0) to (10);
NOTICE: partition constraints are not validated when attaching a readable external table
alter table part attach partition p2_e for values from (10) to (19);
NOTICE: partition constraints are not validated when attaching a readable external table
analyze part;
WARNING: skipping "p2_e" --- cannot analyze this foreign table
WARNING: skipping "p1_e" --- cannot analyze this foreign table
explain select * from part;
QUERY PLAN
------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..514.48 rows=2000000 width=8)
-> Dynamic Foreign Scan on part (cost=0.00..444.93 rows=666667 width=8)
Number of partitions to scan: 2 (out of 2)
Optimizer: Pivotal Optimizer (GPORCA)
(4 rows)
select * from part;
a | b
----+----
1 | 2
3 | 4
5 | 5
7 | 8
11 | 12
13 | 14
15 | 15
17 | 18
(8 rows)
-- test SPE
explain select * from part where b>10 and a>0;
QUERY PLAN
-----------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..462.03 rows=227556 width=8)
-> Dynamic Foreign Scan on part (cost=0.00..455.25 rows=75852 width=8)
Filter: ((b > 10) AND (a > 0))
Number of partitions to scan: 1 (out of 2)
Optimizer: Pivotal Optimizer (GPORCA)
(5 rows)
select * from part where b>10 and a>0;
a | b
----+----
11 | 12
13 | 14
15 | 15
17 | 18
(4 rows)
create table non_part (a int, b int) distributed by (a);
insert into non_part values (15,15);
analyze non_part;
-- mixed partitions with DPE with multiple dynamic scans, select one partition
-- start_ignore
-- unstable test case
explain analyze select * from part, non_part where part.b=non_part.b;
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1163.02 rows=2000000 width=16) (actual time=9.772..9.782 rows=1 loops=1)
-> Hash Join (cost=0.00..1043.77 rows=666667 width=16) (actual time=6.506..8.801 rows=1 loops=1)
Hash Cond: (b = non_part.b)
Extra Text: (seg0) Hash chain length 1.0 avg, 1 max, using 1 of 524288 buckets.
-> Dynamic Foreign Scan on part (cost=0.00..446.40 rows=666667 width=8) (actual time=0.517..0.541 rows=4 loops=1)
Number of partitions to scan: 2 (out of 2)
Partitions scanned: Avg 1.0 x 3 workers. Max 1 parts (seg0).
-> Hash (cost=431.00..431.00 rows=1 width=8) (actual time=0.075..0.080 rows=1 loops=1)
Buckets: 524288 Batches: 1 Memory Usage: 4097kB
-> Partition Selector (selector id: $0) (cost=0.00..431.00 rows=1 width=8) (actual time=0.057..0.060 rows=1 loops=1)
-> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) (actual time=0.026..0.028 rows=1 loops=1)
-> Seq Scan on non_part (cost=0.00..431.00 rows=1 width=8) (actual time=0.034..0.037 rows=1 loops=1)
Planning Time: 33.813 ms
(slice0) Executor memory: 127K bytes.
(slice1) Executor memory: 4309K bytes avg x 3x(0) workers, 4378K bytes max (seg0). Work_mem: 4097K bytes max.
(slice2) Executor memory: 112K bytes avg x 3x(0) workers, 112K bytes max (seg0).
Memory used: 128000kB
Optimizer: Pivotal Optimizer (GPORCA)
Execution Time: 20.437 ms
(19 rows)
-- end_ignore
select * from part, non_part where part.b=non_part.b;
a | b | a | b
----+----+----+----
15 | 15 | 15 | 15
(1 row)
insert into non_part values (5,5);
analyze non_part;
-- mixed partitions with DPE with multiple dynamic scans, select both partition
-- start_ignore
-- unstable test case
explain analyze select * from part, non_part where part.b=non_part.b;
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1163.02 rows=2000000 width=16) (actual time=4.645..4.650 rows=2 loops=1)
-> Hash Join (cost=0.00..1043.77 rows=666667 width=16) (actual time=2.021..4.289 rows=2 loops=1)
Hash Cond: (b = non_part.b)
Extra Text: (seg0) Hash chain length 1.0 avg, 1 max, using 2 of 524288 buckets.
-> Dynamic Foreign Scan on part (cost=0.00..446.40 rows=666667 width=8) (actual time=0.528..0.949 rows=8 loops=1)
Number of partitions to scan: 2 (out of 2)
Partitions scanned: Avg 2.0 x 3 workers. Max 2 parts (seg0).
-> Hash (cost=431.00..431.00 rows=2 width=8) (actual time=0.783..0.786 rows=2 loops=1)
Buckets: 524288 Batches: 1 Memory Usage: 4097kB
-> Partition Selector (selector id: $0) (cost=0.00..431.00 rows=2 width=8) (actual time=0.770..0.776 rows=2 loops=1)
-> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=2 width=8) (actual time=0.757..0.760 rows=2 loops=1)
-> Seq Scan on non_part (cost=0.00..431.00 rows=1 width=8) (actual time=0.037..0.040 rows=1 loops=1)
Planning Time: 32.227 ms
(slice0) Executor memory: 127K bytes.
(slice1) Executor memory: 4382K bytes avg x 3x(0) workers, 4456K bytes max (seg0). Work_mem: 4097K bytes max.
(slice2) Executor memory: 112K bytes avg x 3x(0) workers, 112K bytes max (seg0).
Memory used: 128000kB
Optimizer: Pivotal Optimizer (GPORCA)
Execution Time: 15.233 ms
(19 rows)
-- end_ignore
select * from part, non_part where part.b=non_part.b;
a | b | a | b
----+----+----+----
5 | 5 | 5 | 5
15 | 15 | 15 | 15
(2 rows)
create table p3 (a int, b int) distributed by (a);
create table p4 (a int, b int) distributed by (a);
alter table part attach partition p3 for values from (20) to (30);
alter table part attach partition p4 for values from (30) to (40);
insert into part select i,i from generate_series(25,35)i;
INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner
DETAIL: Falling back to Postgres-based planner because GPORCA does not support the following feature: Insert with External/foreign partition storage types
analyze part;
WARNING: skipping "p2_e" --- cannot analyze this foreign table
WARNING: skipping "p1_e" --- cannot analyze this foreign table
insert into non_part values (32,32);
-- mixed partitions
explain select * from part;
QUERY PLAN
------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..974.67 rows=2000000 width=8)
-> Append (cost=0.00..915.04 rows=666667 width=8)
-> Dynamic Seq Scan on part (cost=0.00..444.93 rows=666667 width=8)
Number of partitions to scan: 2 (out of 4)
-> Dynamic Foreign Scan on part (cost=0.00..444.93 rows=666667 width=8)
Number of partitions to scan: 2 (out of 4)
Optimizer: Pivotal Optimizer (GPORCA)
(7 rows)
select * from part;
a | b
----+----
25 | 25
28 | 28
32 | 32
33 | 33
26 | 26
30 | 30
31 | 31
35 | 35
27 | 27
29 | 29
34 | 34
1 | 2
3 | 4
5 | 5
7 | 8
11 | 12
13 | 14
15 | 15
17 | 18
(19 rows)
--mixed partitions with SPE
explain select * from part where b>10 and b<25;
QUERY PLAN
-----------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..909.71 rows=1 width=8)
-> Result (cost=0.00..909.71 rows=1 width=8)
Filter: ((part.b > 10) AND (part.b < 25))
-> Append (cost=0.00..909.71 rows=1 width=8)
-> Dynamic Seq Scan on part (cost=0.00..444.93 rows=666667 width=8)
Number of partitions to scan: 1 (out of 4)
-> Dynamic Foreign Scan on part (cost=0.00..444.93 rows=666667 width=8)
Number of partitions to scan: 1 (out of 4)
Optimizer: Pivotal Optimizer (GPORCA)
(9 rows)
select * from part where b>10 and b<25 and a > 2;
a | b
----+----
11 | 12
13 | 14
15 | 15
17 | 18
(4 rows)
--mixed partitions with SPE, only do seq scans
explain select * from part where b>25;
QUERY PLAN
------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..529.46 rows=1800010 width=8)
-> Dynamic Seq Scan on part (cost=0.00..475.79 rows=600004 width=8)
Number of partitions to scan: 2 (out of 4)
Filter: (b > 25)
Optimizer: Pivotal Optimizer (GPORCA)
(5 rows)
select * from part where b>25;
a | b
----+----
27 | 27
29 | 29
34 | 34
26 | 26
30 | 30
31 | 31
35 | 35
28 | 28
32 | 32
33 | 33
(10 rows)
--mixed partitions with SPE, only do foreign scans
explain select a from part where b<18;
QUERY PLAN
------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..431.00 rows=1 width=4)
-> Dynamic Foreign Scan on part (cost=0.00..431.00 rows=1 width=8)
Filter: (b < 18)
Number of partitions to scan: 2 (out of 4)
Optimizer: Pivotal Optimizer (GPORCA)
(5 rows)
select a from part where b<18;
a
----
1
3
5
7
11
13
15
(7 rows)
truncate non_part;
-- mixed partitions with DPE with multiple dynamic scans, select no partitions
explain analyze select * from part, non_part where part.b=non_part.b;
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1465.16 rows=1 width=16) (actual time=4.028..4.028 rows=0 loops=1)
-> Hash Join (cost=0.00..1465.16 rows=1 width=16) (actual time=0.000..3.376 rows=0 loops=1)
Hash Cond: (part.b = non_part.b)
-> Append (cost=0.00..915.04 rows=666671 width=8) (never executed)
-> Dynamic Seq Scan on part (cost=0.00..444.93 rows=666671 width=8) (never executed)
Number of partitions to scan: 2 (out of 4)
-> Dynamic Foreign Scan on part (cost=0.00..444.93 rows=666671 width=8) (never executed)
Number of partitions to scan: 2 (out of 4)
-> Hash (cost=431.00..431.00 rows=1 width=8) (actual time=0.000..0.033 rows=0 loops=1)
Buckets: 524288 Batches: 1 Memory Usage: 4096kB
-> Partition Selector (selector id: $0) (cost=0.00..431.00 rows=1 width=8) (actual time=0.000..0.032 rows=0 loops=1)
-> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) (actual time=0.000..0.031 rows=0 loops=1)
-> Seq Scan on non_part (cost=0.00..431.00 rows=1 width=8) (actual time=0.000..0.220 rows=0 loops=1)
Optimizer: Pivotal Optimizer (GPORCA)
Planning Time: 24.170 ms
(slice0) Executor memory: 50K bytes.
(slice1) Executor memory: 4128K bytes avg x 3 workers, 4128K bytes max (seg0). Work_mem: 4096K bytes max.
(slice2) Executor memory: 44K bytes avg x 3 workers, 44K bytes max (seg0).
Memory used: 128000kB
Execution Time: 17.484 ms
(20 rows)
select * from part, non_part where part.b=non_part.b;
a | b | a | b
---+---+---+---
(0 rows)
insert into non_part values (15,15), (32,32);
-- mixed partitions with DPE with multiple dynamic scans, select one partition from each dynamic scan
explain analyze select * from part, non_part where part.b=non_part.b;
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1465.16 rows=1 width=16) (actual time=6.871..8.050 rows=2 loops=1)
-> Hash Join (cost=0.00..1465.16 rows=1 width=16) (actual time=4.864..6.408 rows=1 loops=1)
Hash Cond: (part.b = non_part.b)
Extra Text: (seg0) Hash chain length 1.0 avg, 1 max, using 2 of 524288 buckets.
-> Append (cost=0.00..915.04 rows=666667 width=8) (actual time=0.556..1.762 rows=5 loops=1)
-> Dynamic Seq Scan on part (cost=0.00..444.93 rows=666667 width=8) (actual time=0.343..0.414 rows=3 loops=1)
Number of partitions to scan: 2 (out of 4)
Partitions scanned: Avg 1.0 x 3 workers. Max 1 parts (seg0).
-> Dynamic Foreign Scan on part (cost=0.00..444.93 rows=666667 width=8) (actual time=1.139..1.167 rows=4 loops=1)
Number of partitions to scan: 2 (out of 4)
Partitions scanned: Avg 1.0 x 3 workers. Max 1 parts (seg0).
-> Hash (cost=431.00..431.00 rows=1 width=8) (actual time=1.468..1.469 rows=2 loops=1)
Buckets: 524288 Batches: 1 Memory Usage: 4097kB
-> Partition Selector (selector id: $0) (cost=0.00..431.00 rows=1 width=8) (actual time=0.540..1.464 rows=2 loops=1)
-> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) (actual time=0.534..1.453 rows=2 loops=1)
-> Seq Scan on non_part (cost=0.00..431.00 rows=1 width=8) (actual time=0.855..0.856 rows=1 loops=1)
Optimizer: Pivotal Optimizer (GPORCA)
Planning Time: 15.867 ms
(slice0) Executor memory: 50K bytes.
(slice1) Executor memory: 4257K bytes avg x 3 workers, 4301K bytes max (seg0). Work_mem: 4097K bytes max.
(slice2) Executor memory: 36K bytes avg x 3 workers, 36K bytes max (seg0).
Memory used: 128000kB
Execution Time: 9.566 ms
(23 rows)
select * from part, non_part where part.b=non_part.b;
a | b | a | b
----+----+----+----
32 | 32 | 32 | 32
15 | 15 | 15 | 15
(2 rows)
-- test different servers, should create different scans
-- note: do not select here, just test plan
CREATE EXTENSION file_fdw;
CREATE SERVER file_server1 FOREIGN DATA WRAPPER file_fdw;
CREATE SERVER file_server2 FOREIGN DATA WRAPPER file_fdw;
CREATE FOREIGN TABLE ft1 (
a int,
b int
) SERVER file_server1
OPTIONS ( filename '/does/not/exist.csv', format 'csv');
CREATE FOREIGN TABLE ft2 (
a int,
b int
) SERVER file_server2
OPTIONS ( filename '/does/not/exist.csv', format 'csv');
alter table part attach partition ft1 for values from (40) to (50);
alter table part attach partition ft2 for values from (50) to (60);
analyze part;
ERROR: could not stat file "/does/not/exist.csv": No such file or directory
-- should have multiple dynamic scans
explain select * from part;
QUERY PLAN
------------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2062.88 rows=2000011 width=8)
-> Append (cost=0.00..2003.25 rows=666671 width=8)
-> Dynamic Seq Scan on part (cost=0.00..444.93 rows=666671 width=8)
Number of partitions to scan: 2 (out of 6)
-> Dynamic Foreign Scan on part (cost=0.00..444.93 rows=666671 width=8)
Number of partitions to scan: 2 (out of 6)
-> Redistribute Motion 1:3 (slice2) (cost=0.00..544.11 rows=666671 width=8)
-> Dynamic Foreign Scan on part (cost=0.00..472.80 rows=2000011 width=8)
Number of partitions to scan: 1 (out of 6)
-> Redistribute Motion 1:3 (slice3) (cost=0.00..544.11 rows=666671 width=8)
-> Dynamic Foreign Scan on part (cost=0.00..472.80 rows=2000011 width=8)
Number of partitions to scan: 1 (out of 6)
Optimizer: Pivotal Optimizer (GPORCA)
(11 rows)
-- eliminate unnecessary dynamic scans and foreign sans
explain select * from part where b = 53;
QUERY PLAN
------------------------------------------------------------------
Dynamic Foreign Scan on part (cost=0.00..431.00 rows=1 width=8)
Filter: (b = 53)
Number of partitions to scan: 1 (out of 6)
Optimizer: Pivotal Optimizer (GPORCA)
(4 rows)
-- only select foreign scans
explain select * from part where b > 22;
QUERY PLAN
------------------------------------------------------------------------------------------------
Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1629.96 rows=2000011 width=8)
-> Result (cost=0.00..1570.33 rows=666671 width=8)
Filter: (part.b > 22)
-> Append (cost=0.00..1548.40 rows=666671 width=8)
-> Dynamic Seq Scan on part (cost=0.00..444.93 rows=666671 width=8)
Number of partitions to scan: 2 (out of 6)
-> Redistribute Motion 1:3 (slice2) (cost=0.00..544.11 rows=666671 width=8)
-> Dynamic Foreign Scan on part (cost=0.00..472.80 rows=2000011 width=8)
Number of partitions to scan: 1 (out of 6)
-> Redistribute Motion 1:3 (slice3) (cost=0.00..544.11 rows=666671 width=8)
-> Dynamic Foreign Scan on part (cost=0.00..472.80 rows=2000011 width=8)
Number of partitions to scan: 1 (out of 6)
Optimizer: Pivotal Optimizer (GPORCA)
(13 rows)
--
-- exchange & attach partition
--
alter table part add partition exch1 start(60) end (70);
alter table part add partition exch2 start(70) end (80);
-- exchange with external tables
create external web table p3_e (a int, b int) execute 'cat > @abs_srcdir@/data/part-ext.csv' format 'csv' (delimiter as '|' null as 'null' escape as ' ');
create writable external web table p4_e (a int, b int) execute 'cat > @abs_srcdir@/data/part-ext.csv' format 'csv' (delimiter as '|' null as 'null' escape as ' ');
-- allow exchange readable external table
alter table part exchange partition exch1 with table p3_e;
NOTICE: partition constraints are not validated when attaching a readable external table
-- should disallow writable external table
alter table part exchange partition exch1 with table p4_e;
ERROR: cannot attach a WRITABLE external table
-- exchange with foreign tables
CREATE SERVER file_server3 FOREIGN DATA WRAPPER file_fdw;
CREATE FOREIGN TABLE ft3 (
a int,
b int
) SERVER file_server3
OPTIONS ( filename '/does/not/exist.csv', format 'csv');
-- exchange works, but no error checking like for external tables
alter table part exchange partition exch2 with table ft3;
-- same tests for attach partition
create external web table p5_e (a int, b int) execute 'cat > @abs_srcdir@/data/part-ext.csv' format 'csv' (delimiter as '|' null as 'null' escape as ' ');
create writable external web table p6_e (a int, b int) execute 'cat > @abs_srcdir@/data/part-ext.csv' format 'csv' (delimiter as '|' null as 'null' escape as ' ');
-- allow attach readable external table
alter table part attach partition p5_e for values from (80) to (90);
NOTICE: partition constraints are not validated when attaching a readable external table
-- should disallow writable external table
alter table part attach partition p6_e for values from (90) to (100);
ERROR: cannot attach a WRITABLE external table
-- attach foreign table
CREATE SERVER file_server4 FOREIGN DATA WRAPPER file_fdw;
CREATE FOREIGN TABLE ft4 (
a int,
b int
) SERVER file_server4
OPTIONS ( filename '/does/not/exist.csv', format 'csv');
-- exchange works, but no error checking like for external tables
alter table part attach partition ft4 for values from (100) to (110);