Google Git
Sign in
apache / cloudberry / HEAD / . / src / test / singlenode_isolation2 / expected / gpdispatch_1.out
blob: aed23b485f05cfa29cf4653f996597d815dba2c3 [file] [log] [blame]
-- Try to verify that a session fatal due to OOM should have no effect on other sessions.
-- Report on https://github.com/greenplum-db/gpdb/issues/12399
create extension if not exists gp_inject_fault;
CREATE
1: select gp_inject_fault('make_dispatch_result_error', 'skip', dbid) from gp_segment_configuration where role = 'p' and content = -1;
gp_inject_fault
-----------------
Success:
(1 row)
2: begin;
BEGIN
-- session1 will be fatal.
1: select count(*) > 0 from gp_dist_random('pg_class');
FATAL: could not allocate resources for segworker communication (cdbdisp_async.c:319)
server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
-- session2 should be ok.
2: select count(*) > 0 from gp_dist_random('pg_class');
?column?
----------
t
(1 row)
2: commit;
COMMIT
1q: ... <quitting>
2q: ... <quitting>
select gp_inject_fault('make_dispatch_result_error', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = -1;
gp_inject_fault
-----------------
Success:
(1 row)
--
-- Test case for the WaitEvent of ShareInputScan
--
create table test_waitevent(i int);
CREATE
insert into test_waitevent select generate_series(1,1000);
INSERT 1000
1: set optimizer = off;
SET
1: set gp_cte_sharing to on;
SET
1: select gp_inject_fault_infinite('shareinput_writer_notifyready', 'suspend', 2);
gp_inject_fault_infinite
--------------------------
Success:
(1 row)
1&: WITH a1 as (select * from test_waitevent), a2 as (select * from test_waitevent) SELECT sum(a1.i) FROM a1 INNER JOIN a2 ON a2.i = a1.i UNION ALL SELECT count(a1.i) FROM a1 INNER JOIN a2 ON a2.i = a1.i; <waiting ...>
-- start_ignore
2: copy (select pg_stat_get_activity(NULL) from gp_dist_random('gp_id') where gp_segment_id=0) to '/tmp/_gpdb_test_output.txt';
COPY 9
-- end_ignore
2: select gp_wait_until_triggered_fault('shareinput_writer_notifyready', 1, 2);
gp_wait_until_triggered_fault
-------------------------------
Success:
(1 row)
2: select gp_inject_fault_infinite('shareinput_writer_notifyready', 'resume', 2);
gp_inject_fault_infinite
--------------------------
Success:
(1 row)
2: select gp_inject_fault_infinite('shareinput_writer_notifyready', 'reset', 2);
gp_inject_fault_infinite
--------------------------
Success:
(1 row)
2q: ... <quitting>
1<: <... completed>
sum
--------
500500
1000
(2 rows)
1q: ... <quitting>
!\retcode grep ShareInputScan /tmp/_gpdb_test_output.txt;
-- start_ignore
(100897,9460,10,"",active,"WITH a1 as (select * from test_waitevent), a2 as (select * from test_waitevent) SELECT sum(a1.i) FROM a1 INNER JOIN a2 ON a2.i = a1.i UNION ALL SELECT count(a1.i) FROM a1 INNER JOIN a2 ON a2.i = a1.i;",IPC,ShareInputScan,"Sat Mar 12 23:51:16.151757 2022 PST","Sat Mar 12 23:51:16.151757 2022 PST","Sat Mar 12 23:51:16.14545 2022 PST","Sat Mar 12 23:51:16.151797 2022 PST",127.0.0.1,,63602,,7398,"client backend",f,,,,,,,,f,,f,247,0,unknown)
-- end_ignore
(exited with code 0)
--
-- Test for issue https://github.com/greenplum-db/gpdb/issues/12703
--
-- Case for cdbgang_createGang_async
1: create table t_12703(a int);
CREATE
1:begin;
BEGIN
-- make a cursor so that we have a named portal
1: declare cur12703 cursor for select * from t_12703;
DECLARE
2: select pg_ctl((select datadir from gp_segment_configuration c where c.role='p' and c.content=1), 'stop');
pg_ctl
--------
OK
(1 row)
-- next sql will trigger FTS to mark seg1 as down
2: select gp_request_fts_probe_scan();
gp_request_fts_probe_scan
---------------------------
t
(1 row)
-- this will go to cdbgang_createGang_async's code path
-- for some segments are DOWN. It should not PANIC even
-- with a named portal existing.
1: select * from t_12703;
ERROR: gang was lost due to cluster reconfiguration (cdbgang_async.c:98)
ERROR: Error on receive from seg1 slice1 127.0.1.1:7003 pid=58391: server closed the connection unexpectedly
This probably means the server terminated abnormally
before or while processing the request.
1: abort;
ABORT
1q: ... <quitting>
2q: ... <quitting>
-- Case for cdbCopyEndInternal
-- Provide some data to copy in
4: insert into t_12703 select * from generate_series(1, 10)i;
INSERT 10
4: copy t_12703 to '/tmp/t_12703';
COPY 10
-- make copy in statement hang at the entry point of cdbCopyEndInternal
4: select gp_inject_fault('cdb_copy_end_internal_start', 'suspend', dbid) from gp_segment_configuration where role = 'p' and content = -1;
gp_inject_fault
-----------------
Success:
(1 row)
4q: ... <quitting>
1&: copy t_12703 from '/tmp/t_12703'; <waiting ...>
select gp_wait_until_triggered_fault('cdb_copy_end_internal_start', 1, dbid) from gp_segment_configuration where role = 'p' and content = -1;
gp_wait_until_triggered_fault
-------------------------------
Success:
(1 row)
-- make Gang connection is BAD
select pg_ctl((select datadir from gp_segment_configuration c where c.role='p' and c.content=2), 'stop');
pg_ctl
--------
OK
(1 row)
2: select gp_request_fts_probe_scan();
gp_request_fts_probe_scan
---------------------------
t
(1 row)
2: begin;
BEGIN
select gp_inject_fault('cdb_copy_end_internal_start', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = -1;
gp_inject_fault
-----------------
Success:
(1 row)
-- continue copy it should not PANIC
1<: <... completed>
ERROR: MPP detected 1 segment failures, system is reconnected
1q: ... <quitting>
-- session 2 still alive (means not PANIC happens)
2: select 1;
?column?
----------
1
(1 row)
2: end;
END
2q: ... <quitting>
!\retcode gprecoverseg -aF --no-progress;
-- start_ignore
20211101:14:57:13:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Starting gprecoverseg with args: -aF --no-progress
20211101:14:57:13:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 7.0.0-alpha.0+dev.14950.gaf07f2a6fe build dev'
20211101:14:57:13:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 12beta2 (Apache Cloudberry 7.0.0-alpha.0+dev.14950.gaf07f2a6fe build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0, 64-bit compiled on Nov 1 2021 11:23:12 (with assert checking)'
20211101:14:57:13:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Obtaining Segment details from coordinator...
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Heap checksum setting is consistent between coordinator and the segments that are candidates for recoverseg
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Cloudberry instance recovery parameters
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovery type = Standard
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovery 1 of 2
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Synchronization mode = Full
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance host = zlyu
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance address = zlyu
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance port = 7003
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance host = zlyu
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance address = zlyu
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance port = 7006
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Target = in-place
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovery 2 of 2
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Synchronization mode = Full
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance host = zlyu
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance address = zlyu
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance port = 7004
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance host = zlyu
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance address = zlyu
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance port = 7007
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Target = in-place
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:14:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Starting to create new pg_hba.conf on primary segments
20211101:14:57:15:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Successfully modified pg_hba.conf on primary segments to allow replication connections
20211101:14:57:15:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-2 segment(s) to recover
20211101:14:57:15:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Ensuring 2 failed segment(s) are stopped
20211101:14:57:17:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Validating remote directories
20211101:14:57:18:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Configuring new segments
.
20211101:14:57:20:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Updating configuration with new mirrors
20211101:14:57:20:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Updating mirrors
20211101:14:57:20:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Starting mirrors
20211101:14:57:20:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-era is fbb165f204b577b2_211101145214
20211101:14:57:20:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Commencing parallel segment instance startup, please wait...
20211101:14:57:21:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Process results...
20211101:14:57:21:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-
20211101:14:57:21:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Triggering FTS probe
20211101:14:57:21:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-********************************
20211101:14:57:21:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Segments successfully recovered.
20211101:14:57:21:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-********************************
20211101:14:57:21:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovered mirror segments need to sync WAL with primary segments.
20211101:14:57:21:101597 gprecoverseg:zlyu:gpadmin-[INFO]:-Use 'gpstate -e' to check progress of WAL sync remaining bytes
-- end_ignore
(exited with code 0)
-- loop while segments come in sync
select wait_until_all_segments_synchronized();
wait_until_all_segments_synchronized
--------------------------------------
OK
(1 row)
!\retcode gprecoverseg -ar;
-- start_ignore
20211101:14:57:21:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Starting gprecoverseg with args: -ar
20211101:14:57:21:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 7.0.0-alpha.0+dev.14950.gaf07f2a6fe build dev'
20211101:14:57:21:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 12beta2 (Apache Cloudberry 7.0.0-alpha.0+dev.14950.gaf07f2a6fe build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0, 64-bit compiled on Nov 1 2021 11:23:12 (with assert checking)'
20211101:14:57:21:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Obtaining Segment details from coordinator...
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Cloudberry instance recovery parameters
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovery type = Rebalance
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Unbalanced segment 1 of 4
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance host = zlyu
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance address = zlyu
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance port = 7006
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Balanced role = Mirror
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Current role = Primary
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Unbalanced segment 2 of 4
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance host = zlyu
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance address = zlyu
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance port = 7003
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Balanced role = Primary
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Current role = Mirror
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Unbalanced segment 3 of 4
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance host = zlyu
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance address = zlyu
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance port = 7007
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Balanced role = Mirror
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Current role = Primary
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Unbalanced segment 4 of 4
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance host = zlyu
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance address = zlyu
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Unbalanced instance port = 7004
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Balanced role = Primary
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Current role = Mirror
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Determining primary and mirror segment pairs to rebalance
20211101:14:57:22:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Stopping unbalanced primary segments...
20211101:14:57:23:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Triggering segment reconfiguration
20211101:14:57:28:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Starting segment synchronization
20211101:14:57:28:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-=============================START ANOTHER RECOVER=========================================
20211101:14:57:28:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-local Cloudberry Version: 'postgres (Apache Cloudberry) 7.0.0-alpha.0+dev.14950.gaf07f2a6fe build dev'
20211101:14:57:28:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-coordinator Cloudberry Version: 'PostgreSQL 12beta2 (Apache Cloudberry 7.0.0-alpha.0+dev.14950.gaf07f2a6fe build dev) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0, 64-bit compiled on Nov 1 2021 11:23:12 (with assert checking)'
20211101:14:57:28:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Obtaining Segment details from coordinator...
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Heap checksum setting is consistent between coordinator and the segments that are candidates for recoverseg
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Cloudberry instance recovery parameters
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovery type = Standard
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovery 1 of 2
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Synchronization mode = Incremental
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance host = zlyu
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance address = zlyu
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance port = 7006
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance host = zlyu
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance address = zlyu
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast2/demoDataDir1
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance port = 7003
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Target = in-place
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovery 2 of 2
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Synchronization mode = Incremental
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance host = zlyu
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance address = zlyu
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Failed instance port = 7007
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance host = zlyu
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance address = zlyu
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance directory = /home/gpadmin/workspace/gpdb/gpAux/gpdemo/datadirs/dbfast3/demoDataDir2
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Source instance port = 7004
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:- Recovery Target = in-place
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:----------------------------------------------------------
20211101:14:57:29:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Starting to create new pg_hba.conf on primary segments
20211101:14:57:30:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Successfully modified pg_hba.conf on primary segments to allow replication connections
20211101:14:57:30:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-2 segment(s) to recover
20211101:14:57:30:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Ensuring 2 failed segment(s) are stopped
20211101:14:57:32:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Updating configuration with new mirrors
20211101:14:57:32:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Updating mirrors
20211101:14:57:32:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Running pg_rewind on failed segments
zlyu (dbid 6): pg_rewind: no rewind required
zlyu (dbid 7): pg_rewind: no rewind required
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Starting mirrors
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-era is fbb165f204b577b2_211101145214
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Commencing parallel segment instance startup, please wait...
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Process results...
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Triggering FTS probe
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-********************************
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Segments successfully recovered.
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-********************************
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Recovered mirror segments need to sync WAL with primary segments.
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-Use 'gpstate -e' to check progress of WAL sync remaining bytes
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-==============================END ANOTHER RECOVER==========================================
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-******************************************************************
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-The rebalance operation has completed successfully.
20211101:14:57:36:103065 gprecoverseg:zlyu:gpadmin-[INFO]:-******************************************************************
-- end_ignore
(exited with code 0)
-- loop while segments come in sync
select wait_until_all_segments_synchronized();
wait_until_all_segments_synchronized
--------------------------------------
OK
(1 row)
-- verify no segment is down after recovery
select count(*) from gp_segment_configuration where status = 'd';
count
-------
0
(1 row)