src/test/regress/sql/gp_tablespace_with_faults.source

-- CREATE TABLESPACE
--
-- We need to update the restart point on the mirrors
-- so downstream tests do not attempt to replay records
-- generated by this suite.
--
-- a reload is required.
--
-- start_ignore
\! gpconfig -c create_restartpoint_on_ckpt_record_replay -v on --skipvalidation;
\! gpstop -u;
create extension if not exists plpython3u;
-- end_ignore

create or replace function remove_tablespace_location_directory(tablespace_location_dir text) returns void as $$
	import shutil
	import os
	import glob

	working_directory = os.getcwd()
	result = glob.glob("{pg_tablespace_directory}".format(pg_tablespace_directory=working_directory + "/pg_tblspc"))

	tablespace_symlinks = [
		[dir + '/' + tablespace_oid
			for tablespace_oid in os.listdir(dir)
			] for dir
				in result
				if dir
	]

	tablespace_symlinks_to_remove = [
		tablespace_symlink
			for tablespace_symlink
			in sum(tablespace_symlinks, [])
			if os.readlink(tablespace_symlink).startswith(tablespace_location_dir)]

	for tablespace_symlink_to_remove in tablespace_symlinks_to_remove:
		os.unlink(tablespace_symlink_to_remove);

	try:
		shutil.rmtree(tablespace_location_dir);
	except OSError:
		pass
$$ LANGUAGE plpython3u;


create or replace function setup_tablespace_location_dir_for_test(tablespace_location_dir text) returns void as $$
	import os;
	os.mkdir(tablespace_location_dir);
$$ LANGUAGE plpython3u;


create or replace function disable_fts() returns void as $$
	declare
		master_content_id integer = -1;
		master_primary_role character = 'p';
	begin
		-- intentionally skip fts during these tests
		-- we know we're going to be inducing errors and panics on primaries
		perform gp_inject_fault_infinite('fts_probe', 'skip', dbid)
			from gp_segment_configuration
			where content = master_content_id
			and role = master_primary_role;

		perform gp_request_fts_probe_scan();

	end;
$$ language plpgsql;


create or replace function setup(content_id integer, fault_name text, fault_action_type text, tablespace_dir text) returns void as $$
	begin
		-- reset tablespace directories
		perform remove_tablespace_location_directory(tablespace_dir);
		perform remove_tablespace_location_directory(tablespace_dir) from gp_dist_random('gp_id');
		perform setup_tablespace_location_dir_for_test(tablespace_dir);

		-- setup faults
		perform gp_inject_fault('all', 'reset', dbid) from gp_segment_configuration where content = content_id and role = 'p';
		perform disable_fts();
		perform gp_inject_fault(fault_name, fault_action_type, dbid) from gp_segment_configuration where content = content_id and role = 'p';
	end;
$$ LANGUAGE plpgsql;


create or replace function cleanup(content_id integer, tablespace_location_dir text) returns void as $$
	begin
		CHECKPOINT; -- ensure primary/master xlogs do not leak into following test
		perform force_mirrors_to_catch_up();
		perform gp_inject_fault('all', 'reset', dbid) from gp_segment_configuration;
		perform remove_tablespace_location_directory(tablespace_location_dir);
	end;
$$ LANGUAGE plpgsql;


create or replace function list_tablespace_dbid_dirs(expected_number_of_tablespaces integer, tablespace_location_directory text) returns varchar[] as $$
	import glob
	rows = glob.glob("{tablespace_location_directory}/*".format(tablespace_location_directory=tablespace_location_directory))
	rows.sort()

	found_number_of_tablespaces = len(rows)
	was_success = expected_number_of_tablespaces == found_number_of_tablespaces

	result = []

	if was_success:
		result.append("Success")
	else:
		result.append("Failed")

	result.append('expected {expected_number_of_tablespaces} tablespaces'.format(expected_number_of_tablespaces=expected_number_of_tablespaces))
	result.append('found %d tablespaces' % found_number_of_tablespaces)
	for row in rows:
		result.append(str(row))
	return result
$$ language plpython3u;


create or replace function wait_for_primaries_to_restart() returns boolean as $$
	select count(gp_segment_id) > 0 from gp_dist_random('pg_tablespace');
$$ language sql;


create or replace function list_tablespace_catalog() returns table(gp_segment_id int, tablespace_name text, oid oid) as $$
	select -1 as gp_segment_id, spcname::text, oid from pg_tablespace
	UNION ALL
	select gp_segment_id, spcname::text, oid from gp_dist_random('pg_tablespace')
	order by gp_segment_id, spcname
$$ language sql;


create or replace function list_tablespace_catalog_without_oid() returns table(gp_segment_id int, tablespace_name text) as $$
	select -1 as gp_segment_id, spcname::text from pg_tablespace
	UNION ALL
	select gp_segment_id, spcname::text from gp_dist_random('pg_tablespace')
	order by gp_segment_id, spcname
$$ language sql;


--
-- An error after XLOG_TBLSPC_CREATE on master
--
-- expected: tablespace should not exist after abort
--
\set content_id_under_test -1
\set expected_number_of_tablespaces 0
\set error_type error
\set tablespace_location /tmp/my_tablespace_for_testing
\set fault_to_set after_xlog_create_tablespace

select setup(:content_id_under_test, :'fault_to_set', :'error_type', :'tablespace_location');
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select force_mirrors_to_catch_up();
select * from list_tablespace_catalog();
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');


--
-- An error after XLOG_TBLSPC_CREATE on primary
--
-- expected: tablespace should not exist after abort
--
\set content_id_under_test 0
\set expected_number_of_tablespaces 0
\set error_type error
\set tablespace_location /tmp/my_tablespace_for_testing
\set fault_to_set after_xlog_create_tablespace

select setup(:content_id_under_test, :'fault_to_set', :'error_type', :'tablespace_location');
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select force_mirrors_to_catch_up();
select * from list_tablespace_catalog();
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');


--
-- An panic after XLOG_XACT_PREPARE on primary
--
-- expected: tablespace should not exist after abort
--
\set content_id_under_test 0
\set expected_number_of_tablespaces 0
\set error_type panic
\set tablespace_location /tmp/my_tablespace_for_testing
\set fault_to_set after_xlog_xact_prepare_flushed

select setup(:content_id_under_test, :'fault_to_set', :'error_type', :'tablespace_location');
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select wait_for_primaries_to_restart();
select force_mirrors_to_catch_up();
select * from list_tablespace_catalog();
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');


--
-- An error before XLOG_XACT_PREPARE on primary
--
\set content_id_under_test 0
\set expected_number_of_tablespaces 0
\set error_type error
\set tablespace_location /tmp/my_tablespace_for_testing
\set fault_to_set before_xlog_xact_prepare

select setup(:content_id_under_test, :'fault_to_set', :'error_type', :'tablespace_location');
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select force_mirrors_to_catch_up();
select * from list_tablespace_catalog();
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');

--
-- An error after XLOG_XACT_PREPARE on primary
--
-- This leaves a dangling global transaction in the proc array,
-- but should not happen in practice.
--

--
-- An panic before XLOG_XACT_PREPARE on primary
--
-- expected: tablespaces should be cleaned up everywhere
--           except where the panic happened.
--
-- context:  we no longer have any history of the transaction taking place.
--           it is no longer in memory, and we did not succeed in writing the
--           prepared transaction, therefore we cannot safely clean
--           the tablespace directory.
--           Note that we need to panic even before writing WAL for creating the
--           tablespace so mirror doesn't create the same and results in 2 orphaned
--           directories instead of 1.
--
\set content_id_under_test 0
\set expected_number_of_tablespaces 1
\set error_type panic
\set tablespace_location /tmp/my_tablespace_for_testing
\set fault_to_set before_xlog_create_tablespace

select setup(:content_id_under_test, :'fault_to_set', :'error_type', :'tablespace_location');
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select wait_for_primaries_to_restart();
select force_mirrors_to_catch_up();
select * from list_tablespace_catalog();
-- start_ignore
-- unstable here
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
-- end_ignore
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');


--
-- An unrelated aborted transaction after a `CREATE TABLESPACE`
--    should not effect the created tablespaces.
--
-- expected: tablespace directories should still exist
--
\set tablespace_location /tmp/my_tablespace_for_testing
\set expected_number_of_tablespaces 8
\set content_id_under_test -1

select remove_tablespace_location_directory(:'tablespace_location');
select setup_tablespace_location_dir_for_test(:'tablespace_location');
create tablespace my_tablespace_for_testing LOCATION :'tablespace_location';
select * from force_mirrors_to_catch_up();

-- force an aborted transaction which should no longer see a
--     pending tablespace for deletion
begin;
abort;

-- the tablespace should exist in the catalog
select * from list_tablespace_catalog_without_oid();

-- the tablespace should exist on disk
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');


--
-- An error before XLOG_XACT_DISTRIBUTED_COMMIT on master
--
-- expected: tablespaces should be cleaned up everywhere
--
--
\set content_id_under_test -1
\set expected_number_of_tablespaces 0
\set error_type error
\set tablespace_location /tmp/my_tablespace_for_testing
\set fault_to_set before_xlog_xact_distributed_commit

select setup(:content_id_under_test, :'fault_to_set', :'error_type', :'tablespace_location');
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select force_mirrors_to_catch_up();
select * from list_tablespace_catalog();
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');


--
-- DROP TABLESPACE tests
--
\set tablespace_location '/tmp/my_tablespace_for_testing'
select remove_tablespace_location_directory(:'tablespace_location');
select remove_tablespace_location_directory(:'tablespace_location') from gp_dist_random('gp_id');
select setup_tablespace_location_dir_for_test(:'tablespace_location');

CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
DROP TABLESPACE my_tablespace_for_testing;
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(0, :'tablespace_location');
-- the tablespace should no longer be in the catalog
select * from list_tablespace_catalog_without_oid();
select cleanup(-1, :'tablespace_location');


--
-- An error after XLOG_TBLSPC_DROP
--
--
\set tablespace_location '/tmp/my_tablespace_for_testing'
select remove_tablespace_location_directory(:'tablespace_location');
select remove_tablespace_location_directory(:'tablespace_location') from gp_dist_random('gp_id');
select setup_tablespace_location_dir_for_test(:'tablespace_location');

CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select gp_inject_fault('after_xlog_tblspc_drop', 'error', dbid) from gp_segment_configuration where role = 'p' and content = 0;
DROP TABLESPACE my_tablespace_for_testing;
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(8, :'tablespace_location');
select * from list_tablespace_catalog_without_oid(); -- the tablespace should still exist in the catalog
select gp_inject_fault('all', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = 0;
DROP TABLESPACE my_tablespace_for_testing;
select force_mirrors_to_catch_up();
select cleanup(0, :'tablespace_location');


--
-- An error after XLOG_TBLSPC_DROP on master
--
--
\set tablespace_location '/tmp/my_tablespace_for_testing'
\set content_id_under_test -1

select remove_tablespace_location_directory(:'tablespace_location');
select remove_tablespace_location_directory(:'tablespace_location') from gp_dist_random('gp_id');
select setup_tablespace_location_dir_for_test(:'tablespace_location');


CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select gp_inject_fault('after_xlog_tblspc_drop', 'error', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(8, :'tablespace_location');
select * from list_tablespace_catalog_without_oid(); -- the tablespace should still exist in the catalog
select gp_inject_fault('all', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select force_mirrors_to_catch_up();
select cleanup(:content_id_under_test, :'tablespace_location');

--
-- When the tablespace is not empty, it should not be allowed to be dropped:
--
--
\set tablespace_location '/tmp/my_tablespace_for_testing'
select remove_tablespace_location_directory(:'tablespace_location');
select remove_tablespace_location_directory(:'tablespace_location') from gp_dist_random('gp_id');
select setup_tablespace_location_dir_for_test(:'tablespace_location');

CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
create table foobar (a int) tablespace my_tablespace_for_testing distributed by (a);
DROP TABLESPACE my_tablespace_for_testing;
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(8, :'tablespace_location');
-- the tablespace should still exist in the catalog
select * from list_tablespace_catalog_without_oid(); -- the tablespace should still exist in the catalog
drop table foobar;
drop tablespace my_tablespace_for_testing;
select cleanup(0, :'tablespace_location');


--
-- A panic before XLOG_XACT_COMMIT_PREPARED on primary
--
\set tablespace_location '/tmp/my_tablespace_for_testing'
\set content_id_under_test 0
\set fault_type panic
\set fault_name before_xlog_xact_commit_prepared

select remove_tablespace_location_directory(:'tablespace_location');
select remove_tablespace_location_directory(:'tablespace_location') from gp_dist_random('gp_id');
select setup_tablespace_location_dir_for_test(:'tablespace_location');

select disable_fts();
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select gp_inject_fault(:'fault_name', :'fault_type', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select wait_for_primaries_to_restart();
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(0, :'tablespace_location');
select * from list_tablespace_catalog_without_oid(); -- the tablespace should no longer exist in the catalog
select gp_inject_fault('all', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
select cleanup(:content_id_under_test, :'tablespace_location');


--
-- A panic after XLOG_XACT_PREPARED on primary
--
\set tablespace_location '/tmp/my_tablespace_for_testing'
\set content_id_under_test 0
\set fault_type panic
\set fault_name after_xlog_xact_prepare_flushed

select remove_tablespace_location_directory(:'tablespace_location');
select remove_tablespace_location_directory(:'tablespace_location') from gp_dist_random('gp_id');
select setup_tablespace_location_dir_for_test(:'tablespace_location');

select disable_fts();
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select gp_inject_fault(:'fault_name', :'fault_type', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select wait_for_primaries_to_restart();
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(8, :'tablespace_location');
select * from list_tablespace_catalog_without_oid(); -- the tablespaces should exist in the catalog
select gp_inject_fault('all', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');

--
-- A panic before XLOG_XACT_PREPARED on primary
--
\set tablespace_location '/tmp/my_tablespace_for_testing'
\set content_id_under_test 0
\set fault_type panic
\set fault_name before_xlog_xact_prepare
\set expected_number_of_tablespaces 8

select remove_tablespace_location_directory(:'tablespace_location');
select remove_tablespace_location_directory(:'tablespace_location') from gp_dist_random('gp_id');
select setup_tablespace_location_dir_for_test(:'tablespace_location');

select disable_fts();
CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select gp_inject_fault(:'fault_name', :'fault_type', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
select * from list_tablespace_catalog_without_oid(); -- the tablespaces should exist in the catalog
select gp_inject_fault('all', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');

--
-- An error before XLOG_XACT_PREPARED on primary
--
\set tablespace_location '/tmp/my_tablespace_for_testing'
\set content_id_under_test 0
\set fault_type error
\set fault_name before_xlog_xact_prepare
\set expected_number_of_tablespaces 8

select remove_tablespace_location_directory(:'tablespace_location');
select remove_tablespace_location_directory(:'tablespace_location') from gp_dist_random('gp_id');
select setup_tablespace_location_dir_for_test(:'tablespace_location');

CREATE TABLESPACE my_tablespace_for_testing LOCATION :'tablespace_location';
select gp_inject_fault(:'fault_name', :'fault_type', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select force_mirrors_to_catch_up();
select list_tablespace_dbid_dirs(:expected_number_of_tablespaces, :'tablespace_location');
select * from list_tablespace_catalog_without_oid(); -- the tablespaces should exist in the catalog
select gp_inject_fault('all', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = :content_id_under_test;
DROP TABLESPACE my_tablespace_for_testing;
select cleanup(:content_id_under_test, :'tablespace_location');

--
-- An error after XLOG_XACT_DISTRIBUTED_COMMIT on master
-- tested manually, as a crash causes test to stop running.
-- Success, expected 0 tablespaces, got 0 tablespaces
--

--
-- Disable this feature as it is not the default behavior
--
-- a reload is required.
--
-- start_ignore
\! gpconfig -r create_restartpoint_on_ckpt_record_replay --skipvalidation;
\! gpstop -u;
-- end_ignore