| /*------------------------------------------------------------------------- |
| * |
| * cdb_restore.c |
| * cdb_restore is a utility for restoring a cdb cluster of postgres databases |
| * |
| *------------------------------------------------------------------------- |
| */ |
| |
| #include "postgres_fe.h" |
| #include <assert.h> |
| #include <unistd.h> |
| #include <signal.h> |
| #include <regex.h> |
| #include <ctype.h> |
| #include <pthread.h> |
| #include "getopt_long.h" |
| #include "pqexpbuffer.h" |
| #include "libpq-fe.h" |
| #include "fe-auth.h" |
| #include "pg_backup.h" |
| #include "cdb_table.h" |
| #include "cdb_dump_util.h" |
| #include "cdb_backup_state.h" |
| #include "cdb_dump.h" |
| |
| |
| /* This is necessary on platforms where optreset variable is not available. |
| * Look at "man getopt" documentation for details. |
| */ |
| #ifndef HAVE_OPTRESET |
| int optreset; |
| #endif |
| |
| |
| /* Forward decls */ |
| static void freeThreadParmArray(ThreadParmArray * pParmAr); |
| static bool createThreadParmArray(int nCount, ThreadParmArray * pParmAr); |
| static char *addPassThroughParm(char Parm, const char *pszValue, char *pszPassThroughParmString); |
| static bool fillInputOptions(int argc, char **argv, InputOptions * pInputOpts); |
| static bool parmValNeedsQuotes(const char *Value); |
| static void usage(void); |
| static void *threadProc(void *arg); |
| static void myHandler(SIGNAL_ARGS); |
| static void installSignalHandlers(void); |
| static bool restoreMaster(InputOptions * pInputOpts, PGconn *pConn, SegmentDatabase *sSegDB, SegmentDatabase *tSegDB, ThreadParm * pParm); |
| static void spinOffThreads(PGconn *pConn, InputOptions * pInputOpts, const const RestorePairArray * restorePair, ThreadParmArray * pParmAr); |
| static int reportRestoreResults(const char *pszReportDirectory, const ThreadParm * pMasterParm, const ThreadParmArray * pParmAr); |
| static int reportMasterError(InputOptions inputopts, const ThreadParm * pMasterParm, const char *localMsg); |
| static void updateAppendOnlyStats(PGconn *pConn); |
| static bool g_b_SendCancelMessage = false; |
| typedef struct option optType; |
| |
| static RestoreOptions *opts = NULL; |
| static bool dataOnly = false; |
| static bool schemaOnly = false; |
| static bool bForcePassword = false; |
| static bool bIgnoreVersion = false; |
| static const char *pszAgent = "gp_restore_agent"; |
| |
| static const char *logInfo = "INFO"; |
| static const char *logWarn = "WARN"; |
| static const char *logError = "ERROR"; |
| static const char *logFatal = "FATAL"; |
| const char *progname; |
| |
| |
| int |
| main(int argc, char **argv) |
| { |
| PGconn *pConn = NULL; |
| int failCount = -1; |
| int i; |
| bool found_master; |
| SegmentDatabase *sourceSegDB = NULL; |
| SegmentDatabase *targetSegDB = NULL; |
| |
| /* This struct holds the values of the command line parameters */ |
| InputOptions inputOpts; |
| |
| /* |
| * This struct holds an array of the segment databases from the master mpp |
| * tables |
| */ |
| RestorePairArray restorePairAr; |
| |
| /* This struct holds an array of the thread parameters */ |
| ThreadParmArray parmAr; |
| ThreadParm masterParm; |
| |
| memset(&inputOpts, 0, sizeof(inputOpts)); |
| memset(&restorePairAr, 0, sizeof(restorePairAr)); |
| memset(&parmAr, 0, sizeof(parmAr)); |
| memset(&masterParm, 0, sizeof(masterParm)); |
| |
| progname = get_progname(argv[0]); |
| |
| /* Parse command line for options */ |
| if (!fillInputOptions(argc, argv, &inputOpts)) |
| { |
| failCount = reportMasterError(inputOpts, &masterParm, |
| get_early_error()); |
| mpp_err_msg(logInfo, progname, "Reporting Master Error from fillInputOptions.\n"); |
| goto cleanup; |
| } |
| |
| |
| mpp_err_msg(logInfo, progname, "Analyzed command line options.\n"); |
| |
| /* Connect to database on the master */ |
| mpp_err_msg(logInfo, progname, "Connecting to master segment on host %s port %s database %s.\n", |
| StringNotNull(inputOpts.pszPGHost, "localhost"), |
| StringNotNull(inputOpts.pszPGPort, "5432"), |
| StringNotNull(inputOpts.pszMasterDBName, "?")); |
| |
| pConn = GetMasterConnection(progname, inputOpts.pszMasterDBName, inputOpts.pszPGHost, |
| inputOpts.pszPGPort, inputOpts.pszUserName, |
| bForcePassword, bIgnoreVersion, true); |
| if (pConn == NULL) |
| { |
| masterParm.pOptionsData = &inputOpts; |
| failCount = reportMasterError(inputOpts, &masterParm, |
| get_early_error()); |
| goto cleanup; |
| |
| } |
| |
| /* Read mpp segment databases configuration from the master */ |
| mpp_err_msg(logInfo, progname, "Reading Greenplum Database configuration info from master segment database.\n"); |
| if (!GetRestoreSegmentDatabaseArray(pConn, &restorePairAr, inputOpts.backupLocation, inputOpts.pszRawDumpSet, dataOnly)) |
| goto cleanup; |
| |
| /* find master node */ |
| targetSegDB = NULL; |
| found_master = false; |
| |
| for (i = 0; i < restorePairAr.count; i++) |
| { |
| targetSegDB = &restorePairAr.pData[i].segdb_target; |
| sourceSegDB = &restorePairAr.pData[i].segdb_source; |
| |
| if (targetSegDB->role == ROLE_MASTER) |
| { |
| found_master = true; |
| break; |
| } |
| } |
| |
| /* Install the SIGINT and SIGTERM handlers */ |
| installSignalHandlers(); |
| |
| /* |
| * restore master first. However, if we restore only data, or if master |
| * segment is not included in our restore set, skip master |
| */ |
| if (!dataOnly && found_master) |
| { |
| if (!restoreMaster(&inputOpts, pConn, sourceSegDB, targetSegDB, &masterParm)) |
| { |
| failCount = reportMasterError(inputOpts, &masterParm, NULL); |
| goto cleanup; |
| } |
| |
| } |
| |
| /* restore segdbs. However, if we restore schema only, skip segdbs */ |
| if (!schemaOnly) |
| { |
| if (restorePairAr.count > 0) |
| { |
| /* |
| * Create the threads to talk to each of the databases being |
| * restored. Wait for them to finish. |
| */ |
| spinOffThreads(pConn, &inputOpts, &restorePairAr, &parmAr); |
| |
| mpp_err_msg(logInfo, progname, "All remote %s programs are finished.\n", pszAgent); |
| } |
| |
| /* |
| * If any AO table data was restored, update the master AO statistics |
| */ |
| updateAppendOnlyStats(pConn); |
| } |
| |
| /* |
| * restore post-data items last. However, if we restore only data, or if master |
| * segment is not included in our restore set, skip this step |
| */ |
| if (!dataOnly && found_master) |
| { |
| char * newParms; |
| int newlen = strlen(" --post-data-schema-only") + 1; |
| if (inputOpts.pszPassThroughParms != NULL) |
| newlen += strlen(inputOpts.pszPassThroughParms); |
| newParms = malloc(newlen); |
| newParms[0] = '\0'; |
| if (inputOpts.pszPassThroughParms != NULL) |
| strcpy(newParms, inputOpts.pszPassThroughParms); |
| strcat(newParms, " --post-data-schema-only"); |
| inputOpts.pszPassThroughParms = newParms; |
| |
| if (!restoreMaster(&inputOpts, pConn, sourceSegDB, targetSegDB, &masterParm)) |
| { |
| failCount = reportMasterError(inputOpts, &masterParm, NULL); |
| goto cleanup; |
| } |
| |
| } |
| |
| |
| /* Produce results report */ |
| failCount = reportRestoreResults(inputOpts.pszReportDirectory, &masterParm, &parmAr); |
| |
| cleanup: |
| FreeInputOptions(&inputOpts); |
| if (opts != NULL) |
| free(opts); |
| FreeRestorePairArray(&restorePairAr); |
| freeThreadParmArray(&parmAr); |
| |
| if (masterParm.pszErrorMsg) |
| free(masterParm.pszErrorMsg); |
| |
| if (masterParm.pszRemoteBackupPath) |
| free(masterParm.pszRemoteBackupPath); |
| |
| if (pConn != NULL) |
| PQfinish(pConn); |
| |
| return (failCount == 0 ? 0 : 1); |
| } |
| |
| static void |
| usage(void) |
| { |
| printf(("%s restores a Greenplum Database database from an archive created by gp_dump.\n\n"), progname); |
| printf(("Usage:\n")); |
| printf((" %s [OPTIONS]\n"), progname); |
| |
| printf(("\nGeneral options:\n")); |
| printf((" -d, --dbname=NAME output database name\n")); |
| /* printf((" -f, --file=FILENAME output file name\n")); */ |
| /* printf((" -F, --format=c|t specify backup file format\n")); */ |
| printf((" -i, --ignore-version proceed even when server version mismatches\n")); |
| //printf((" -l, --list print summarized TOC of the archive\n")); |
| printf((" -v, --verbose verbose mode. adds verbose information to the\n" |
| " per segment status files\n")); |
| printf((" --help show this help, then exit\n")); |
| printf((" --version output version information, then exit\n")); |
| |
| printf(("\nOptions controlling the output content:\n")); |
| printf((" -a, --data-only restore only the data, no schema\n")); |
| //printf((" -c, --clean clean (drop) schema prior to create\n")); |
| //printf((" -C, --create issue commands to create the database\n")); |
| //printf((" -I, --index=NAME restore named index\n")); |
| //printf((" -L, --use-list=FILENAME use specified table of contents for ordering\n" |
| // " output from this file\n")); |
| //printf((" -N, --orig-order restore in original dump order\n")); |
| //printf((" -o, --oid-order restore in OID order\n")); |
| //printf((" -O, --no-owner do not output commands to set object ownership\n")); |
| //printf((" -P, --function=NAME(args)\n" |
| // " restore named function\n")); |
| //printf((" -r, --rearrange rearrange output to put indexes etc. at end\n")); |
| printf((" -s, --schema-only restore only the schema, no data\n")); |
| //printf((" -S, --superuser=NAME specify the superuser user name to use for\n" |
| // " disabling triggers\n")); |
| //printf((" -t, --table=NAME restore named table\n")); |
| //printf((" -T, --trigger=NAME restore named trigger\n")); |
| //printf((" -x, --no-privileges skip restoration of access privileges (grant/revoke)\n")); |
| //printf((" -X disable-triggers, --disable-triggers\n" |
| // " disable triggers during data-only restore\n")); |
| |
| printf(("\nConnection options:\n")); |
| printf((" -h, --host=HOSTNAME database server host or socket directory\n")); |
| printf((" -p, --port=PORT database server port number\n")); |
| printf((" -U, --username=NAME connect as specified database user\n")); |
| printf((" -W, --password force password prompt (should happen automatically)\n")); |
| |
| printf(("\nGreenplum Database specific options:\n")); |
| printf((" --gp-c use gunzip for in-line de-compression\n")); |
| printf((" --gp-d=BACKUPFILEDIR directory where backup files are located\n")); |
| printf((" --gp-i ignore error\n")); |
| printf((" --gp-k=KEY date time backup key from gp_backup run\n")); |
| printf((" --gp-r=REPORTFILEDIR directory where report file is placed\n")); |
| printf((" --gp-l=FILELOCATIONS backup files are on (p)rimaries only (default)\n")); |
| printf((" or (i)ndividual segdb (must be followed with a list of dbid's\n")); |
| printf((" where backups are located. For example: --gp-l=i[10,12,15]\n")); |
| |
| //printf(("\nIf no input file name is supplied, then standard input is used.\n\n")); |
| //printf(("Report bugs to <pgsql-bugs@postgresql.org>.\n")); |
| } |
| |
| bool |
| fillInputOptions(int argc, char **argv, InputOptions * pInputOpts) |
| { |
| int c; |
| /* Archive *AH; */ |
| /* char *inputFileSpec; */ |
| extern int optind; |
| extern char *optarg; |
| /* static int use_setsessauth = 0; */ |
| static int disable_triggers = 0; |
| bool bSawCDB_S_Option; |
| int lenCmdLineParms; |
| int i; |
| |
| |
| struct option cmdopts[] = { |
| /*{"clean", 0, NULL, 'c'}, */ |
| /* {"create", 0, NULL, 'C'}, */ |
| {"data-only", 0, NULL, 'a'}, |
| {"dbname", 1, NULL, 'd'}, |
| /*{"file", 1, NULL, 'f'}, |
| {"format", 1, NULL, 'F'},*/ |
| //{"function", 1, NULL, 'P'}, |
| {"host", 1, NULL, 'h'}, |
| {"ignore-version", 0, NULL, 'i'}, |
| /* {"index", 1, NULL, 'I'}, */ |
| /* {"list", 0, NULL, 'l'}, */ |
| /* {"no-privileges", 0, NULL, 'x'}, */ |
| {"no-acl", 0, NULL, 'x'}, |
| /* {"no-owner", 0, NULL, 'O'}, */ |
| {"no-reconnect", 0, NULL, 'R'}, |
| {"port", 1, NULL, 'p'}, |
| /* {"oid-order", 0, NULL, 'o'}, */ |
| /* {"orig-order", 0, NULL, 'N'}, */ |
| {"password", 0, NULL, 'W'}, |
| /* {"rearrange", 0, NULL, 'r'}, */ |
| {"schema-only", 0, NULL, 's'}, |
| /* {"superuser", 1, NULL, 'S'}, */ |
| /* {"table", 1, NULL, 't'}, */ |
| /* {"trigger", 1, NULL, 'T'}, */ |
| /* {"use-list", 1, NULL, 'L'}, */ |
| {"username", 1, NULL, 'U'}, |
| {"verbose", 0, NULL, 'v'}, |
| |
| /* |
| * the following options don't have an equivalent short option letter, |
| * but are available as '-X long-name' |
| */ |
| //{"use-set-session-authorization", no_argument, &use_setsessauth, 1}, |
| //{"disable-triggers", no_argument, &disable_triggers, 1}, |
| |
| /* |
| * the following are Greenplum Database specific, and don't have an equivalent short option |
| */ |
| {"gp-c", no_argument, NULL, 1}, |
| /* {"gp-cf", required_argument, NULL, 2}, */ |
| {"gp-d", required_argument, NULL, 3}, |
| {"gp-i", no_argument, NULL, 4}, |
| /* {"gp-hdb", required_argument, NULL, 5}, */ |
| {"gp-k", required_argument, NULL, 6}, |
| {"gp-r", required_argument, NULL, 7}, |
| {"gp-s", required_argument, NULL, 8}, |
| {"gp-l", required_argument, NULL, 9}, |
| |
| {NULL, 0, NULL, 0} |
| }; |
| |
| /* Initialize option fields */ |
| memset(pInputOpts, 0, sizeof(InputOptions)); |
| |
| pInputOpts->actors = SET_NO_MIRRORS; /* restore primaries only. |
| * mirrors will get sync'ed |
| * automatically */ |
| pInputOpts->backupLocation = FILE_ON_PRIMARIES; |
| pInputOpts->bOnErrorStop = true; |
| pInputOpts->pszBackupDirectory = NULL; |
| |
| opts = (RestoreOptions *) calloc(1, sizeof(RestoreOptions)); |
| if (opts == NULL) |
| { |
| mpp_err_msg_cache(logError, progname, "error allocating memory for RestoreOptions"); |
| return false; |
| } |
| |
| opts->format = archUnknown; |
| opts->suppressDumpWarnings = false; |
| |
| bSawCDB_S_Option = false; |
| |
| if (argc == 1) |
| { |
| usage(); |
| exit(0); |
| } |
| |
| if (argc > 1) |
| { |
| if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) |
| { |
| usage(); |
| exit(0); |
| } |
| if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) |
| { |
| puts("gp_restore (Greenplum Database) " GP_VERSION); |
| exit(0); |
| } |
| } |
| |
| /* |
| * Record the command line parms as a string for documentation in the |
| * ouput report |
| */ |
| lenCmdLineParms = 0; |
| for (i = 1; i < argc; i++) |
| { |
| if (i > 1) |
| lenCmdLineParms += 1; |
| |
| lenCmdLineParms += strlen(argv[i]); |
| if (parmValNeedsQuotes(argv[i])) |
| lenCmdLineParms += 2; |
| } |
| |
| pInputOpts->pszCmdLineParms = (char *) malloc(lenCmdLineParms + 1); |
| if (pInputOpts->pszCmdLineParms == NULL) |
| { |
| mpp_err_msg_cache(logError, progname, "error allocating memory for pInputOpts->pszCmdLineParms\n"); |
| return false; |
| } |
| |
| memset(pInputOpts->pszCmdLineParms, 0, lenCmdLineParms + 1); |
| for (i = 1; i < argc; i++) |
| { |
| if (i > 1) |
| strcat(pInputOpts->pszCmdLineParms, " "); |
| if (parmValNeedsQuotes(argv[i])) |
| { |
| strcat(pInputOpts->pszCmdLineParms, "\""); |
| strcat(pInputOpts->pszCmdLineParms, argv[i]); |
| strcat(pInputOpts->pszCmdLineParms, "\""); |
| } |
| else |
| strcat(pInputOpts->pszCmdLineParms, argv[i]); |
| } |
| |
| while ((c = getopt_long(argc, argv, "acd:h:ip:RsuU:vwW", |
| cmdopts, NULL)) != -1) |
| { |
| switch (c) |
| { |
| case 'a': /* Dump data only */ |
| opts->dataOnly = 1; |
| dataOnly = true; |
| pInputOpts->pszPassThroughParms = addPassThroughParm(c, NULL, pInputOpts->pszPassThroughParms); |
| break; |
| /* case 'c': */ /* clean (i.e., drop) schema prior to create */ |
| /* opts->dropSchema = 1; |
| pInputOpts->pszPassThroughParms = addPassThroughParm(c, NULL, pInputOpts->pszPassThroughParms); |
| break; |
| */ |
| /* case 'C': |
| opts->create = 1; |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, NULL, pInputOpts->pszPassThroughParms ); |
| break; |
| */ |
| case 'd': |
| opts->dbname = strdup(optarg); |
| pInputOpts->pszDBName = Safe_strdup(opts->dbname); |
| pInputOpts->pszMasterDBName = Safe_strdup(opts->dbname); |
| break; |
| /* case 'f': // output file name |
| opts->filename = strdup(optarg); |
| break; |
| |
| case 'F': |
| if (strlen(optarg) != 0) |
| { |
| opts->formatName = strdup(optarg); |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, optarg, pInputOpts->pszPassThroughParms ); |
| } |
| |
| break; |
| */ |
| case 'h': |
| if (strlen(optarg) != 0) |
| { |
| opts->pghost = strdup(optarg); |
| pInputOpts->pszPGHost = Safe_strdup(optarg); |
| } |
| |
| break; |
| case 'i': |
| /* obsolete option */ |
| break; |
| |
| /* case 'l': // * Dump the TOC summary * */ |
| /* opts->tocSummary = 1; */ |
| /* pInputOpts->pszPassThroughParms = addPassThroughParm( c, NULL, pInputOpts->pszPassThroughParms ); */ |
| /* break; */ |
| |
| /* case 'L': // * input TOC summary file name * */ |
| /* opts->tocFile = strdup(optarg); */ |
| /* break; */ |
| |
| /* case 'N': |
| opts->origOrder = 1; |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, NULL, pInputOpts->pszPassThroughParms ); |
| break; |
| case 'o': |
| opts->oidOrder = 1; |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, NULL, pInputOpts->pszPassThroughParms ); |
| break; |
| case 'O': |
| opts->noOwner = 1; |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, NULL, pInputOpts->pszPassThroughParms ); |
| break; |
| */ case 'p': |
| if (strlen(optarg) != 0) |
| { |
| opts->pgport = strdup(optarg); |
| pInputOpts->pszPGPort = Safe_strdup(optarg); |
| } |
| break; |
| /* case 'r': |
| opts->rearrange = 1; |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, NULL, pInputOpts->pszPassThroughParms ); |
| break; |
| */ case 'R': |
| /* no-op, still accepted for backwards compatibility */ |
| break; |
| /* case 'P': // Function |
| opts->selTypes = 1; |
| opts->selFunction = 1; |
| opts->functionNames = strdup(optarg); |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, optarg, pInputOpts->pszPassThroughParms ); |
| break; |
| case 'I': // Index |
| opts->selTypes = 1; |
| opts->selIndex = 1; |
| opts->indexNames = strdup(optarg); |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, optarg, pInputOpts->pszPassThroughParms ); |
| break; |
| case 'T': // Trigger |
| opts->selTypes = 1; |
| opts->selTrigger = 1; |
| opts->triggerNames = strdup(optarg); |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, optarg, pInputOpts->pszPassThroughParms ); |
| break; |
| */ case 's': /* dump schema only */ |
| opts->schemaOnly = 1; |
| schemaOnly = true; |
| pInputOpts->pszPassThroughParms = addPassThroughParm(c, NULL, pInputOpts->pszPassThroughParms); |
| break; |
| /* case 'S': // * Superuser username * */ |
| /* if (strlen(optarg) != 0) */ |
| /* opts->superuser = strdup(optarg); */ |
| /* break; */ |
| /* case 't': // Dump data for this table only |
| opts->selTypes = 1; |
| opts->selTable = 1; |
| opts->tableNames = strdup(optarg); |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, optarg, pInputOpts->pszPassThroughParms ); |
| break; |
| */ |
| case 'u': |
| opts->promptPassword = TRI_YES; |
| opts->username = simple_prompt("User name: ", 100, true); |
| pInputOpts->pszUserName = Safe_strdup(opts->username); |
| break; |
| |
| case 'U': |
| opts->username = optarg; |
| pInputOpts->pszUserName = Safe_strdup(opts->username); |
| break; |
| |
| case 'v': /* verbose */ |
| opts->verbose = 1; |
| pInputOpts->pszPassThroughParms = addPassThroughParm(c, NULL, pInputOpts->pszPassThroughParms); |
| break; |
| |
| case 'w': |
| opts->promptPassword = TRI_NO; |
| break; |
| |
| case 'W': |
| opts->promptPassword = TRI_YES; |
| break; |
| |
| /* case 'x': // skip ACL dump |
| opts->aclsSkip = 1; |
| pInputOpts->pszPassThroughParms = addPassThroughParm( c, NULL, pInputOpts->pszPassThroughParms ); |
| break; |
| |
| case 'X': |
| if (strcmp(optarg, "use-set-session-authorization") == 0) |
| * *//* no-op, still allowed |
| for compatibility */ ; |
| |
| /* else if (strcmp(optarg, "disable-triggers") == 0) */ |
| /* disable_triggers = 1; */ |
| /* else */ |
| /* { */ |
| /* fprintf(stderr, */ |
| /* ("%s: invalid -X option -- %s\n"), */ |
| /* progname , optarg); */ |
| |
| /* |
| * fprintf(stderr, ("Try \"%s --help\" for more |
| * information.\n"), progname ); |
| */ |
| /* exit(1); */ |
| /* } */ |
| |
| /* |
| * pInputOpts->pszPassThroughParms = addPassThroughParm( c, |
| * optarg, pInputOpts->pszPassThroughParms ); |
| */ |
| /* break; */ |
| |
| /* This covers the long options equivalent to -X xxx. */ |
| case 0: |
| break; |
| |
| case 1: |
| /* gp-c remote compression program */ |
| pInputOpts->pszCompressionProgram = "gunzip"; /* Safe_strdup(optarg); */ |
| break; |
| |
| case 2: |
| /* gp-cf control file */ |
| |
| /* |
| * temporary disabled if ( bSawCDB_S_Option ) { Ignore gp_s |
| * option |
| * |
| * mpp_err_msg(logInfo, progname, "ignoring the gp-s |
| * option, since the gp-cf option is set.\n"); } |
| * |
| * if (!FillCDBSet(&pInputOpts->set, optarg)) exit(1); |
| */ |
| |
| break; |
| |
| case 3: |
| /* gp-d backup remote directory */ |
| pInputOpts->pszBackupDirectory = Safe_strdup(optarg); |
| break; |
| |
| case 4: |
| /* gp-e on error stop */ |
| pInputOpts->bOnErrorStop = false; |
| break; |
| |
| case 5: |
| /* gp-hdb master database name */ |
| /* pInputOpts->pszMasterDBName = Safe_strdup(optarg); */ |
| break; |
| |
| case 6: |
| /* gp-k backup timestamp key */ |
| pInputOpts->pszKey = Safe_strdup(optarg); |
| break; |
| |
| case 7: |
| /* gp-r report directory */ |
| pInputOpts->pszReportDirectory = Safe_strdup(optarg); |
| break; |
| case 8: |
| /* gp-s backup set specification */ |
| |
| /* |
| * temporay disabled bSawCDB_S_Option = true; if ( |
| * pInputOpts->set.type == SET_INDIVIDUAL ) { Ignore this, as |
| * we've already encountered an individual spec |
| * mpp_err_msg(logInfo, progname, "ignoring the gp-s |
| * option, since the gp-cf option is set.\n"); } else |
| * |
| * |
| * |
| * { if ( strcasecmp( optarg, "a") == 0 ) pInputOpts->set.type |
| * = SET_ALL; else if ( strcasecmp( optarg, "t") == 0 ) |
| * pInputOpts->set.type = SET_SEGDBS_ONLY; else if ( |
| * strcasecmp( optarg, "h") == 0 ) pInputOpts->set.type = |
| * SET_MASTER_ONLY; else { mpp_err_msg(logInfo, progname, |
| * "invalid gp-s option %s. Must be a, t, or h.\n", optarg); |
| * exit(1); } } break; |
| */ |
| case 9: |
| |
| /* gp-l backup file location */ |
| if (strcasecmp(optarg, "p") == 0) |
| { |
| pInputOpts->backupLocation = FILE_ON_PRIMARIES; |
| pInputOpts->pszRawDumpSet = NULL; |
| } |
| else if (strncasecmp(optarg, "i", 1) == 0) |
| { |
| pInputOpts->backupLocation = FILE_ON_INDIVIDUAL; |
| pInputOpts->pszRawDumpSet = Safe_strdup(optarg); |
| } |
| else |
| { |
| mpp_err_msg_cache(logError, progname, "invalid gp-l option %s. Must be p (on primary segments), or i[dbid list] for \"individual\".\n", optarg); |
| return false; |
| } |
| break; |
| |
| default: |
| mpp_err_msg_cache(logError, progname, "Try \"%s --help\" for more information.\n", progname); |
| return false; |
| } |
| } |
| |
| /* |
| * get PG env variables, override only of no cmd-line value specified |
| */ |
| if (pInputOpts->pszDBName == NULL) |
| { |
| if (getenv("PGDATABASE") != NULL) |
| { |
| pInputOpts->pszDBName = Safe_strdup(getenv("PGDATABASE")); |
| pInputOpts->pszMasterDBName = Safe_strdup(getenv("PGDATABASE")); /* TODO: is this variable redundant? */ |
| } |
| } |
| |
| if (pInputOpts->pszPGPort == NULL) |
| { |
| if (getenv("PGPORT") != NULL) |
| pInputOpts->pszPGPort = Safe_strdup(getenv("PGPORT")); |
| } |
| |
| if (pInputOpts->pszPGHost == NULL) |
| { |
| if (getenv("PGHOST") != NULL) |
| pInputOpts->pszPGHost = Safe_strdup(getenv("PGHOST")); |
| } |
| |
| |
| /* |
| * Check for cmd line parameter errors. |
| */ |
| if (pInputOpts->pszDBName == NULL) |
| { |
| mpp_err_msg_cache(logError, progname, "%s command line missing the database parameter (-d)\n", progname); |
| return false; |
| } |
| |
| opts->useDB = 1; |
| |
| if (pInputOpts->pszKey == NULL) |
| { |
| mpp_err_msg_cache(logError, progname, "%s command line missing the date time backup key parameter (--gp-k)\n", progname); |
| return false; |
| } |
| |
| if (pInputOpts->pszMasterDBName == NULL) |
| { |
| mpp_err_msg_cache(logError, progname, "%s command line missing the master database parameter (--)\n", progname); |
| return false; |
| } |
| |
| if ((dataOnly || schemaOnly) && pInputOpts->backupLocation == FILE_ON_INDIVIDUAL) |
| { |
| mpp_err_msg_cache(logError, progname, "options \"schema only\" (-s) or \"data only\" (-a) cannot be used together with --gp-s=i\n" |
| "If you want to use --gp-i and dump only data, omit dbid number 1 (master) from the individual dbid list.\n"); |
| return false; |
| } |
| |
| |
| if (opts->formatName == NULL) |
| { |
| opts->formatName = "p"; |
| } |
| |
| opts->disable_triggers = disable_triggers; |
| bForcePassword = (opts->promptPassword == TRI_YES); |
| /* bIgnoreVersion = (opts->ignoreVersion == 1); */ |
| if (opts->formatName) |
| { |
| |
| switch (opts->formatName[0]) |
| { |
| |
| case 'c': |
| case 'C': |
| opts->format = archCustom; |
| break; |
| |
| case 'f': |
| case 'F': |
| opts->format = archFiles; |
| break; |
| |
| case 't': |
| case 'T': |
| opts->format = archTar; |
| break; |
| |
| case 'p': |
| case 'P': |
| opts->format = archNull; |
| break; |
| |
| default: |
| mpp_err_msg_cache(logError, progname, "unrecognized archive format '%s'; please specify 'p', 't', or 'c'\n", |
| opts->formatName); |
| return false; |
| } |
| } |
| |
| return true; |
| |
| } |
| |
| /* |
| * parmValNeedsQuotes: This function checks to see whether there is any whitespace in the parameter value. |
| * This is used for pass thru parameters, top know whether to enclose them in quotes or not. |
| */ |
| bool |
| parmValNeedsQuotes(const char *Value) |
| { |
| static regex_t rFinder; |
| static bool bCompiled = false; |
| static bool bAlwaysPutQuotes = false; |
| bool bPutQuotes = false; |
| |
| if (!bCompiled) |
| { |
| if (0 != regcomp(&rFinder, "[[:space:]]", REG_EXTENDED | REG_NOSUB)) |
| bAlwaysPutQuotes = true; |
| |
| bCompiled = true; |
| } |
| |
| if (!bAlwaysPutQuotes) |
| { |
| if (regexec(&rFinder, Value, 0, NULL, 0) == 0) |
| bPutQuotes = true; |
| } |
| else |
| bPutQuotes = true; |
| |
| return bPutQuotes; |
| } |
| |
| /* |
| * addPassThroughParm: this function adds to the string of pass-through parameters. These get sent to each |
| * backend and get passed to the gp_dump program. |
| */ |
| char * |
| addPassThroughParm(char Parm, const char *pszValue, char *pszPassThroughParmString) |
| { |
| char *pszRtn; |
| bool bFirstTime = (pszPassThroughParmString == NULL); |
| |
| if (pszValue != NULL) |
| { |
| if (parmValNeedsQuotes(pszValue)) |
| { |
| if (bFirstTime) |
| pszRtn = MakeString("-%c \"%s\"", Parm, pszValue); |
| else |
| pszRtn = MakeString("%s -%c \"%s\"", pszPassThroughParmString, Parm, pszValue); |
| } |
| else |
| { |
| if (bFirstTime) |
| pszRtn = MakeString("-%c %s", Parm, pszValue); |
| else |
| pszRtn = MakeString("%s -%c %s", pszPassThroughParmString, Parm, pszValue); |
| } |
| } |
| else |
| { |
| if (bFirstTime) |
| pszRtn = MakeString("-%c", Parm); |
| else |
| pszRtn = MakeString("%s -%c", pszPassThroughParmString, Parm); |
| } |
| |
| return pszRtn; |
| } |
| |
| /* threadProc: This function is used to connect to one database that needs to be restored, |
| * make a gp_restore_launch call to cause the restore process to begin on the instance hosting the |
| * database. Then it waits for notifications from the process. It receives a notification |
| * wnenever the gp_restore_agent process signals with a NOTIFY. This happens when the process starts, |
| * and when it finishes. |
| */ |
| void * |
| threadProc(void *arg) |
| { |
| /* |
| * The argument is a pointer to a ThreadParm structure that stays around |
| * for the entire time the program is running. so we need not worry about |
| * making a copy of it. |
| */ |
| ThreadParm *pParm = (ThreadParm *) arg; |
| |
| SegmentDatabase *sSegDB = pParm->pSourceSegDBData; |
| SegmentDatabase *tSegDB = pParm->pTargetSegDBData; |
| const InputOptions *pInputOpts = pParm->pOptionsData; |
| const char *pszKey = pInputOpts->pszKey; |
| PGconn *pConn; |
| char *pszPassThroughCredentials; |
| char *pszPassThroughTargetInfo; |
| PQExpBuffer Qry; |
| PGresult *pRes; |
| int sock; |
| fd_set input_mask; |
| bool bSentCancelMessage; |
| bool bIsFinished; |
| bool bIsStarted; |
| int nTries; |
| char *pszNotifyRelName; |
| char *pszNotifyRelNameStart; |
| char *pszNotifyRelNameSucceed; |
| char *pszNotifyRelNameFail; |
| struct timeval tv; |
| PGnotify *pNotify; |
| int nNotifies; |
| |
| /* |
| * Block SIGINT and SIGKILL signals (we can handle them in the main |
| * thread) |
| */ |
| sigset_t newset; |
| |
| sigemptyset(&newset); |
| sigaddset(&newset, SIGINT); |
| sigaddset(&newset, SIGKILL); |
| pthread_sigmask(SIG_SETMASK, &newset, NULL); |
| |
| /* Create a connection for this thread */ |
| if (strcasecmp(pInputOpts->pszDBName, pInputOpts->pszMasterDBName) != 0) |
| { |
| if (sSegDB->pszDBName != NULL) |
| free(sSegDB->pszDBName); |
| |
| sSegDB->pszDBName = Safe_strdup(pInputOpts->pszDBName); |
| } |
| |
| /* connect to the source segDB to start gp_dump_agent there */ |
| pConn = MakeDBConnection(sSegDB, false); |
| if (PQstatus(pConn) == CONNECTION_BAD) |
| { |
| g_b_SendCancelMessage = true; |
| pParm->pszErrorMsg = MakeString("Connection to dbid %d on host %s failed: %s", |
| sSegDB->dbid, StringNotNull(sSegDB->pszHost, "localhost"), PQerrorMessage(pConn)); |
| mpp_err_msg_cache(logError, progname, pParm->pszErrorMsg); |
| PQfinish(pConn); |
| return NULL; |
| } |
| |
| /* issue a LISTEN command for 3 different names */ |
| DoCancelNotifyListen(pConn, true, pszKey, sSegDB->role, sSegDB->dbid, tSegDB->dbid, SUFFIX_START); |
| DoCancelNotifyListen(pConn, true, pszKey, sSegDB->role, sSegDB->dbid, tSegDB->dbid, SUFFIX_SUCCEED); |
| DoCancelNotifyListen(pConn, true, pszKey, sSegDB->role, sSegDB->dbid, tSegDB->dbid, SUFFIX_FAIL); |
| |
| mpp_err_msg(logInfo, progname, "Listening for messages from dbid %d server (source) for dbid %d restore\n", sSegDB->dbid, tSegDB->dbid); |
| |
| /* Execute gp_restore_launch */ |
| |
| /* |
| * If there is a password associated with this login, we pass it as a |
| * base64 encoded string in the parameter pszPassThroughCredentials |
| */ |
| pszPassThroughCredentials = NULL; |
| if (sSegDB->pszDBPswd != NULL && *sSegDB->pszDBPswd != '\0') |
| pszPassThroughCredentials = DataToBase64(sSegDB->pszDBPswd, strlen(sSegDB->pszDBPswd)); |
| |
| pszPassThroughTargetInfo = MakeString("--target-dbid %d --target-host %s --target-port %d ", |
| tSegDB->dbid, tSegDB->pszHost, tSegDB->port); |
| |
| Qry = createPQExpBuffer(); |
| appendPQExpBuffer(Qry, "SELECT * FROM gp_restore_launch('%s', '%s', '%s', '%s', '%s', '%s', %d, %s)", |
| StringNotNull(pInputOpts->pszBackupDirectory, ""), |
| pszKey, |
| StringNotNull(pInputOpts->pszCompressionProgram, ""), |
| StringNotNull(pInputOpts->pszPassThroughParms, ""), |
| StringNotNull(pszPassThroughCredentials, ""), |
| StringNotNull(pszPassThroughTargetInfo, ""), |
| tSegDB->dbid, |
| pInputOpts->bOnErrorStop ? "true" : "false"); |
| |
| if (pszPassThroughCredentials != NULL) |
| free(pszPassThroughCredentials); |
| |
| if (pszPassThroughTargetInfo != NULL) |
| free(pszPassThroughTargetInfo); |
| |
| pRes = PQexec(pConn, Qry->data); |
| if (!pRes || PQresultStatus(pRes) != PGRES_TUPLES_OK || PQntuples(pRes) == 0) |
| { |
| g_b_SendCancelMessage = true; |
| pParm->pszErrorMsg = MakeString("could not start Greenplum Database restore: %s", PQerrorMessage(pConn)); |
| mpp_err_msg_cache(logError, progname, pParm->pszErrorMsg); |
| PQfinish(pConn); |
| return NULL; |
| } |
| |
| mpp_err_msg(logInfo, progname, "Successfully launched Greenplum Database restore on dbid %d to restore dbid %d\n", sSegDB->dbid, tSegDB->dbid); |
| |
| pParm->pszRemoteBackupPath = strdup(PQgetvalue(pRes, 0, 0)); |
| |
| PQclear(pRes); |
| destroyPQExpBuffer(Qry); |
| |
| /* Now wait for notifications from the back end back end */ |
| sock = PQsocket(pConn); |
| |
| bSentCancelMessage = false; |
| bIsFinished = false; |
| bIsStarted = false; |
| nTries = 0; |
| |
| pszNotifyRelName = MakeString("N%s_%d_%d_T%d", |
| pszKey, sSegDB->role, sSegDB->dbid, tSegDB->dbid); |
| pszNotifyRelNameStart = MakeString("%s_%s", pszNotifyRelName, SUFFIX_START); |
| pszNotifyRelNameSucceed = MakeString("%s_%s", pszNotifyRelName, SUFFIX_SUCCEED); |
| pszNotifyRelNameFail = MakeString("%s_%s", pszNotifyRelName, SUFFIX_FAIL); |
| |
| while (!bIsFinished) |
| { |
| /* |
| * Check to see whether another thread has failed and therefore we |
| * should cancel |
| */ |
| if (g_b_SendCancelMessage && !bSentCancelMessage && bIsStarted) |
| { |
| mpp_err_msg(logInfo, progname, "noticed that a cancel order is in effect. Informing dbid %d on host %s\n", |
| sSegDB->dbid, StringNotNull(sSegDB->pszHost, "localhost")); |
| |
| /* |
| * Either one of the other threads have failed, or a Ctrl C was |
| * received. So post a cancel message |
| */ |
| DoCancelNotifyListen(pConn, false, pszKey, sSegDB->role, sSegDB->dbid, tSegDB->dbid, NULL); |
| bSentCancelMessage = true; |
| } |
| |
| tv.tv_sec = 2; |
| tv.tv_usec = 0; |
| FD_ZERO(&input_mask); |
| FD_SET(sock, &input_mask); |
| if (select(sock + 1, &input_mask, NULL, NULL, &tv) < 0) |
| { |
| g_b_SendCancelMessage = true; |
| pParm->pszErrorMsg = MakeString("select failed for backup key %s, source dbid %d, target dbid %d failed\n", |
| pszKey, sSegDB->dbid, tSegDB->dbid); |
| mpp_err_msg(logError, progname, pParm->pszErrorMsg); |
| PQfinish(pConn); |
| return NULL; |
| } |
| |
| /* See whether the connection went down */ |
| if (PQstatus(pConn) == CONNECTION_BAD) |
| { |
| g_b_SendCancelMessage = true; |
| pParm->pszErrorMsg = MakeString("connection went down for backup key %s, source dbid %d, target dbid %d\n", |
| pszKey, sSegDB->dbid, tSegDB->dbid); |
| mpp_err_msg(logFatal, progname, pParm->pszErrorMsg); |
| PQfinish(pConn); |
| return NULL; |
| } |
| |
| PQconsumeInput(pConn); |
| |
| nNotifies = 0; |
| while (NULL != (pNotify = PQnotifies(pConn))) |
| { |
| if (strncasecmp(pszNotifyRelName, pNotify->relname, strlen(pszNotifyRelName)) == 0) |
| { |
| nNotifies++; |
| if (strcasecmp(pszNotifyRelNameStart, pNotify->relname) == 0) |
| { |
| bIsStarted = true; |
| mpp_err_msg(logInfo, progname, "restore started for source dbid %d, target dbid %d on host %s\n", |
| sSegDB->dbid, tSegDB->dbid, StringNotNull(sSegDB->pszHost, "localhost")); |
| } |
| else if (strcasecmp(pszNotifyRelNameSucceed, pNotify->relname) == 0) |
| { |
| bIsFinished = true; |
| pParm->bSuccess = true; |
| mpp_err_msg(logInfo, progname, "restore succeeded for source dbid %d, target dbid %d on host %s\n", |
| sSegDB->dbid, tSegDB->dbid, StringNotNull(sSegDB->pszHost, "localhost")); |
| } |
| else if (strcasecmp(pszNotifyRelNameFail, pNotify->relname) == 0) |
| { |
| g_b_SendCancelMessage = true; |
| bIsFinished = true; |
| pParm->bSuccess = false; |
| /* Make call to get error message from file on server */ |
| pParm->pszErrorMsg = ReadBackendBackupFile(pConn, pInputOpts->pszBackupDirectory, pszKey, BFT_RESTORE_STATUS, progname); |
| |
| mpp_err_msg(logError, progname, "restore failed for source dbid %d, target dbid %d on host %s\n", |
| sSegDB->dbid, tSegDB->dbid, StringNotNull(sSegDB->pszHost, "localhost")); |
| } |
| } |
| |
| PQfreemem(pNotify); |
| } |
| |
| if (nNotifies == 0) |
| { |
| if (!bIsStarted) |
| { |
| nTries++; |
| /* increase timeout to 10min for heavily loaded system */ |
| if (nTries == 300) |
| { |
| bIsFinished = true; |
| pParm->bSuccess = false; |
| pParm->pszErrorMsg = MakeString("restore failed to start for source dbid %d, target dbid %d on host %s in the required time interval\n", |
| sSegDB->dbid, tSegDB->dbid, StringNotNull(sSegDB->pszHost, "localhost")); |
| DoCancelNotifyListen(pConn, false, pszKey, sSegDB->role, sSegDB->dbid, tSegDB->dbid, NULL); |
| bSentCancelMessage = true; |
| } |
| } |
| } |
| } |
| |
| if (pszNotifyRelName != NULL) |
| free(pszNotifyRelName); |
| if (pszNotifyRelNameStart != NULL) |
| free(pszNotifyRelNameStart); |
| if (pszNotifyRelNameSucceed != NULL) |
| free(pszNotifyRelNameSucceed); |
| if (pszNotifyRelNameFail != NULL) |
| free(pszNotifyRelNameFail); |
| |
| PQfinish(pConn); |
| |
| return (NULL); |
| } |
| |
| /* |
| * installSignalHandlers: This function sets both the SIGINT and SIGTERM signal handlers |
| * to the routine myHandler. |
| */ |
| void |
| installSignalHandlers(void) |
| { |
| struct sigaction act, |
| oact; |
| |
| act.sa_handler = myHandler; |
| sigemptyset(&act.sa_mask); |
| act.sa_flags = 0; |
| act.sa_flags |= SA_RESTART; |
| |
| /* Install SIGINT interrupt handler */ |
| if (sigaction(SIGINT, &act, &oact) < 0) |
| { |
| mpp_err_msg_cache(logError, progname, "Error trying to set SIGINT interrupt handler\n"); |
| } |
| |
| act.sa_handler = myHandler; |
| sigemptyset(&act.sa_mask); |
| act.sa_flags = 0; |
| act.sa_flags |= SA_RESTART; |
| |
| /* Install SIGTERM interrupt handler */ |
| if (sigaction(SIGTERM, &act, &oact) < 0) |
| { |
| mpp_err_msg_cache(logError, progname, "Error trying to set SIGTERM interrupt handler\n"); |
| } |
| } |
| |
| /* |
| * myHandler: This function is the signal handler for both SIGINT and SIGTERM signals. |
| * It simply sets a global variable, which is checked by each thread to see whether it |
| * should cancel the restore running on the remote database. |
| */ |
| void |
| myHandler(SIGNAL_ARGS) |
| { |
| static bool bAlreadyHere = false; |
| |
| if (bAlreadyHere) |
| { |
| return; |
| } |
| |
| bAlreadyHere = true; |
| |
| g_b_SendCancelMessage = true; |
| } |
| |
| bool |
| restoreMaster(InputOptions * pInputOpts, |
| PGconn *pConn, |
| SegmentDatabase *sSegDB, |
| SegmentDatabase *tSegDB, |
| ThreadParm * pParm) |
| { |
| bool rtn = false; |
| bool bForcePassword = false; |
| /*bool bIgnoreVersion = false;*/ |
| |
| memset(pParm, 0, sizeof(ThreadParm)); |
| |
| /* We need to restore the master */ |
| mpp_err_msg(logInfo, progname, "Starting to restore the master database.\n"); |
| |
| if (opts != NULL && opts->promptPassword == TRI_YES) |
| bForcePassword = true; |
| |
| /* Now, spin off a thread to do the restore, and wait for it to finish */ |
| pParm->thread = 0; |
| pParm->pTargetSegDBData = tSegDB; |
| pParm->pSourceSegDBData = sSegDB; |
| pParm->pOptionsData = pInputOpts; |
| pParm->bSuccess = false; |
| pParm->pszErrorMsg = NULL; |
| pParm->pszRemoteBackupPath = NULL; |
| |
| mpp_err_msg(logInfo, progname, "Creating thread to restore master database: host %s port %d database %s\n", |
| StringNotNull(pParm->pTargetSegDBData->pszHost, "localhost"), |
| pParm->pTargetSegDBData->port, |
| pParm->pTargetSegDBData->pszDBName |
| ); |
| |
| pthread_create(&pParm->thread, |
| NULL, |
| threadProc, |
| pParm); |
| |
| pthread_join(pParm->thread, NULL); |
| pParm->thread = (pthread_t) 0; |
| |
| if (pParm->bSuccess) |
| { |
| mpp_err_msg(logInfo, progname, "Successfully restored master database: host %s port %d database %s\n", |
| StringNotNull(pParm->pTargetSegDBData->pszHost, "localhost"), |
| pParm->pTargetSegDBData->port, |
| pParm->pTargetSegDBData->pszDBName |
| ); |
| rtn = true; |
| } |
| else |
| { |
| if (pParm->pszErrorMsg != NULL) |
| mpp_err_msg(logError, progname, "see error report for details\n"); |
| else |
| { |
| mpp_err_msg(logError, progname, "Failed to restore master database: host %s port %d database %s\n", |
| StringNotNull(pParm->pTargetSegDBData->pszHost, "localhost"), |
| pParm->pTargetSegDBData->port, |
| pParm->pTargetSegDBData->pszDBName |
| ); |
| } |
| } |
| |
| return rtn; |
| } |
| |
| /* |
| * updateAppendOnlyStats |
| * |
| * for every append only that exists in the database (whether we just restored |
| * it or not) update its statistics on the master - note that this must not be |
| * made with utility mode connection, but rather a regular connection. |
| */ |
| void |
| updateAppendOnlyStats(PGconn *pConn) |
| { |
| /* query that gets all ao tables in the database */ |
| PQExpBuffer get_query = createPQExpBuffer(); |
| PGresult *get_res; |
| int get_ntups = 0; |
| |
| /* query that updates all ao tables in the database */ |
| PQExpBuffer update_query = createPQExpBuffer(); |
| PGresult *update_res = NULL; |
| int update_ntups = 0; |
| |
| /* misc */ |
| int i_tablename = 0; |
| int i_schemaname = 0; |
| int i = 0; |
| |
| mpp_err_msg(logInfo, progname, "updating Append Only table statistics\n"); |
| |
| /* Fetch all Append Only tables */ |
| appendPQExpBuffer(get_query, |
| "SELECT c.relname,n.nspname " |
| "FROM pg_class c, pg_namespace n " |
| "WHERE c.relnamespace=n.oid " |
| "AND (c.relstorage='a' OR c.relstorage='c')"); |
| |
| get_res = PQexec(pConn, get_query->data); |
| |
| if (!get_res || PQresultStatus(get_res) != PGRES_TUPLES_OK) |
| { |
| const char *err; |
| |
| if (get_res) |
| err = PQresultErrorMessage(get_res); |
| else |
| err = PQerrorMessage(pConn); |
| |
| mpp_err_msg(logWarn, progname, |
| "Failed to get Append Only tables for updating stats. " |
| "error was: %s\n", err); |
| |
| } |
| else |
| { |
| /* A-OK */ |
| get_ntups = PQntuples(get_res); |
| i_tablename = PQfnumber(get_res, "relname"); |
| i_schemaname = PQfnumber(get_res, "nspname"); |
| |
| for (i = 0; i < get_ntups; i++) |
| { |
| char *tablename = PQgetvalue(get_res, i, i_tablename); |
| char *schemaname = PQgetvalue(get_res, i, i_schemaname); |
| |
| resetPQExpBuffer(update_query); |
| appendPQExpBuffer(update_query, |
| "SELECT * " |
| "FROM gp_update_ao_master_stats('%s.%s')", |
| schemaname, tablename); |
| |
| update_res = PQexec(pConn, update_query->data); |
| |
| if (!update_res || PQresultStatus(update_res) != PGRES_TUPLES_OK) |
| { |
| const char *err; |
| |
| if (update_res) |
| err = PQresultErrorMessage(update_res); |
| else |
| err = PQerrorMessage(pConn); |
| |
| mpp_err_msg(logWarn, progname, |
| "Failed to update Append Only table stats. error " |
| "was: %s, query was: %s\n", err, update_query->data); |
| } |
| else |
| { |
| /* A-OK */ |
| update_ntups = PQntuples(update_res); |
| if (update_ntups != 1) |
| mpp_err_msg(logWarn, progname, |
| "Expected 1 tuple, got %d. query was: %s\n", |
| update_ntups, update_query->data); |
| } |
| } |
| } |
| |
| /* clean up */ |
| if (update_res) |
| PQclear(update_res); |
| |
| if (get_res) |
| PQclear(get_res); |
| |
| destroyPQExpBuffer(get_query); |
| destroyPQExpBuffer(update_query); |
| |
| } |
| |
| /* |
| * spinOffThreads: This function deals with all the threads that drive the backend restores. |
| * First, it initializes the ThreadParmArray. Then it loops and creates each of the threads. |
| * It then waits for all threads to finish. |
| */ |
| void |
| spinOffThreads(PGconn *pConn, InputOptions * pInputOpts, const RestorePairArray * restorePairAr, ThreadParmArray * pParmAr) |
| { |
| int i; |
| |
| /* |
| * Create a thread and have it work on executing the dump on the |
| * appropriate segdb. |
| */ |
| if (!createThreadParmArray(restorePairAr->count, pParmAr)) |
| { |
| mpp_err_msg(logError, progname, "Cannot allocate memory for thread parameters\n"); |
| exit(-1); |
| } |
| |
| for (i = 0; i < pParmAr->count; i++) |
| { |
| ThreadParm *pParm = &pParmAr->pData[i]; |
| |
| pParm->pSourceSegDBData = &restorePairAr->pData[i].segdb_source; |
| pParm->pTargetSegDBData = &restorePairAr->pData[i].segdb_target; |
| pParm->pOptionsData = pInputOpts; |
| pParm->bSuccess = false; |
| |
| /* exclude master node */ |
| if (pParm->pTargetSegDBData->role == ROLE_MASTER) |
| continue; |
| |
| mpp_err_msg(logInfo, progname, "Creating thread to restore dbid %d (%s:%d) from backup file on dbid %d (%s:%d)\n", |
| pParm->pTargetSegDBData->dbid, |
| StringNotNull(pParm->pTargetSegDBData->pszHost, "localhost"), |
| pParm->pTargetSegDBData->port, |
| pParm->pSourceSegDBData->dbid, |
| StringNotNull(pParm->pSourceSegDBData->pszHost, "localhost"), |
| pParm->pSourceSegDBData->port); |
| |
| pthread_create(&pParm->thread, |
| NULL, |
| threadProc, |
| pParm); |
| } |
| |
| mpp_err_msg(logInfo, progname, "Waiting for all remote %s programs to finish.\n", pszAgent); |
| |
| /* Wait for all threads to complete */ |
| for (i = 0; i < pParmAr->count; i++) |
| { |
| ThreadParm *pParm = &pParmAr->pData[i]; |
| |
| /* exclude master node and not valid node */ |
| if (pParm->thread == (pthread_t) 0) |
| continue; |
| pthread_join(pParm->thread, NULL); |
| pParm->thread = (pthread_t) 0; |
| } |
| } |
| |
| /* |
| * freeThreadParmArray: This function frees the memory allocated for the pData element of the |
| * ThreadParmArray, as well as the strings allocated within each element |
| * of the array. It does not free the pParmAr itself. |
| */ |
| void |
| freeThreadParmArray(ThreadParmArray * pParmAr) |
| { |
| int i; |
| |
| for (i = 0; i < pParmAr->count; i++) |
| { |
| ThreadParm *p = &pParmAr->pData[i]; |
| |
| if (p->pszRemoteBackupPath != NULL) |
| free(p->pszRemoteBackupPath); |
| |
| if (p->pszErrorMsg != NULL) |
| free(p->pszErrorMsg); |
| } |
| |
| if (pParmAr->pData != NULL) |
| free(pParmAr->pData); |
| |
| pParmAr->count = 0; |
| pParmAr->pData = NULL; |
| } |
| |
| /* |
| * createThreadParmArray: This function initializes the count and pData elements of the ThreadParmArray. |
| */ |
| bool |
| createThreadParmArray(int nCount, ThreadParmArray * pParmAr) |
| { |
| int i; |
| |
| pParmAr->count = nCount; |
| pParmAr->pData = (ThreadParm *) malloc(nCount * sizeof(ThreadParm)); |
| if (pParmAr->pData == NULL) |
| return false; |
| |
| for (i = 0; i < nCount; i++) |
| { |
| ThreadParm *p = &pParmAr->pData[i]; |
| |
| p->thread = 0; |
| p->pSourceSegDBData = NULL; |
| p->pTargetSegDBData = NULL; |
| p->pOptionsData = NULL; |
| p->bSuccess = true; |
| p->pszErrorMsg = NULL; |
| p->pszRemoteBackupPath = NULL; |
| } |
| |
| return true; |
| } |
| |
| /* |
| * reportRestoreResults: This function outputs a summary of the restore results to a file or to stdout |
| */ |
| int |
| reportRestoreResults(const char *pszReportDirectory, const ThreadParm * pMasterParm, const ThreadParmArray * pParmAr) |
| { |
| InputOptions *pOptions; |
| PQExpBuffer reportBuf = NULL; |
| FILE *fRptFile = NULL; |
| char *pszReportPathName = NULL; |
| char *pszFormat; |
| int i; |
| int failCount; |
| const ThreadParm *pParm; |
| char *pszStatus; |
| char *pszMsg; |
| |
| assert(pParmAr != NULL && pMasterParm != NULL); |
| |
| if (pParmAr->count == 0 && pMasterParm->pOptionsData == NULL) |
| { |
| mpp_err_msg(logInfo, progname, "No results to report."); |
| return 0; |
| } |
| |
| pOptions = (pParmAr->count > 0) ? pParmAr->pData[0].pOptionsData : pMasterParm->pOptionsData; |
| assert(pOptions != NULL); |
| |
| |
| /* |
| * Set the report directory if not set by user (with --gp-r) |
| */ |
| if (pszReportDirectory == NULL || *pszReportDirectory == '\0') |
| { |
| |
| if (getenv("MASTER_DATA_DIRECTORY") != NULL) |
| { |
| /* |
| * report directory not set by user - default to |
| * $MASTER_DATA_DIRECTORY |
| */ |
| pszReportDirectory = Safe_strdup(getenv("MASTER_DATA_DIRECTORY")); |
| } |
| else |
| { |
| /* $MASTER_DATA_DIRECTORY not set. Default to current directory */ |
| pszReportDirectory = "./"; |
| } |
| } |
| |
| /* See whether we can create the file in the report directory */ |
| if (pszReportDirectory[strlen(pszReportDirectory) - 1] != '/') |
| pszFormat = "%s/gp_restore_%s.rpt"; |
| else |
| pszFormat = "%sgp_restore_%s.rpt"; |
| |
| pszReportPathName = MakeString(pszFormat, pszReportDirectory, pOptions->pszKey); |
| |
| fRptFile = fopen(pszReportPathName, "w"); |
| if (fRptFile == NULL) |
| mpp_err_msg(logWarn, progname, "Cannot open report file %s for writing. Will use stdout instead.\n", |
| pszReportPathName); |
| |
| /* buffer to store the complete restore report */ |
| reportBuf = createPQExpBuffer(); |
| |
| /* |
| * Write to buffer a report showing the timestamp key, what the command |
| * line options were, which segment databases were backed up, to where, |
| * and any errors that occurred. |
| */ |
| appendPQExpBuffer(reportBuf, "\nGreenplum Database Restore Report\n"); |
| appendPQExpBuffer(reportBuf, "Timestamp Key: %s\n", pOptions->pszKey); |
| |
| appendPQExpBuffer(reportBuf, "gp_restore Command Line: %s\n", |
| StringNotNull(pOptions->pszCmdLineParms, "None")); |
| |
| appendPQExpBuffer(reportBuf, "Pass through Command Line Options: %s\n", |
| StringNotNull(pOptions->pszPassThroughParms, "None")); |
| |
| appendPQExpBuffer(reportBuf, "Compression Program: %s\n", |
| StringNotNull(pOptions->pszCompressionProgram, "None")); |
| |
| appendPQExpBuffer(reportBuf, "\n"); |
| appendPQExpBuffer(reportBuf, "Individual Results\n"); |
| |
| failCount = 0; |
| for (i = 0; i < pParmAr->count + 1; i++) |
| { |
| if (i == 0) |
| { |
| if (pMasterParm->pOptionsData != NULL) |
| pParm = pMasterParm; |
| else |
| continue; |
| } |
| else |
| { |
| pParm = &pParmAr->pData[i - 1]; |
| if (pParm->pTargetSegDBData->role == ROLE_MASTER) |
| continue; |
| } |
| |
| if (!pParm->bSuccess) |
| failCount++; |
| pszStatus = pParm->bSuccess ? "Succeeded" : "Failed with error: \n{\n"; |
| pszMsg = pParm->pszErrorMsg; |
| if (pszMsg == NULL) |
| pszMsg = ""; |
| |
| appendPQExpBuffer(reportBuf, "\tRestore of %s on dbid %d (%s:%d) from %s: %s%s%s\n", |
| pParm->pTargetSegDBData->pszDBName, |
| pParm->pTargetSegDBData->dbid, |
| StringNotNull(pParm->pTargetSegDBData->pszHost, "localhost"), |
| pParm->pTargetSegDBData->port, |
| StringNotNull(pParm->pszRemoteBackupPath, ""), |
| pszStatus, |
| pszMsg, |
| (strcmp(pszMsg, "") == 0) ? "" : "}" |
| ); |
| } |
| |
| if (failCount == 0) |
| appendPQExpBuffer(reportBuf, "\n%s utility finished successfully.\n", progname); |
| else |
| appendPQExpBuffer(reportBuf, "\n%s utility finished unsuccessfully with %d failures.\n", progname, failCount); |
| |
| /* write report to report file */ |
| if (fRptFile != NULL) |
| { |
| mpp_msg(logInfo, progname, "Report results also written to %s.\n", pszReportPathName); |
| fprintf(fRptFile, "%s", reportBuf->data); |
| fclose(fRptFile); |
| } |
| |
| /* write report to stdout */ |
| fprintf(stdout, "%s", reportBuf->data); |
| |
| if (pszReportPathName != NULL) |
| free(pszReportPathName); |
| |
| destroyPQExpBuffer(reportBuf); |
| |
| return failCount; |
| } |
| |
| /* |
| * the same as reportRestoreResults but only for the master. This is a little ugly |
| * since we repeat some of the code but we need a majoy overhaul in dump/restore |
| * anyhow - so it will do for now. |
| */ |
| int |
| reportMasterError(InputOptions inputopts, const ThreadParm * pMasterParm, const char *localMsg) |
| { |
| InputOptions *pOptions; |
| PQExpBuffer reportBuf = NULL; |
| FILE *fRptFile = NULL; |
| char *pszReportPathName = NULL; |
| char *pszFormat; |
| int failCount; |
| const ThreadParm *pParm; |
| char *pszStatus; |
| char *pszMsg; |
| char *pszReportDirectory; |
| |
| assert(pMasterParm != NULL); |
| |
| if (pMasterParm->pOptionsData == NULL) |
| { |
| pOptions = &inputopts; |
| } |
| else |
| { |
| pOptions = pMasterParm->pOptionsData; |
| } |
| |
| /* generate a timestamp key if we haven't already */ |
| if (!pOptions->pszKey) |
| pOptions->pszKey = GenerateTimestampKey(); |
| |
| /* |
| * Set the report directory if not set by user (with --gp-r) |
| */ |
| pszReportDirectory = pOptions->pszReportDirectory; |
| if (pszReportDirectory == NULL || *pszReportDirectory == '\0') |
| { |
| |
| if (getenv("MASTER_DATA_DIRECTORY") != NULL) |
| { |
| /* |
| * report directory not set by user - default to |
| * $MASTER_DATA_DIRECTORY |
| */ |
| pszReportDirectory = Safe_strdup(getenv("MASTER_DATA_DIRECTORY")); |
| } |
| else |
| { |
| /* $MASTER_DATA_DIRECTORY not set. Default to current directory */ |
| pszReportDirectory = "./"; |
| } |
| } |
| |
| /* See whether we can create the file in the report directory */ |
| if (pszReportDirectory[strlen(pszReportDirectory) - 1] != '/') |
| pszFormat = "%s/gp_restore_%s.rpt"; |
| else |
| pszFormat = "%sgp_restore_%s.rpt"; |
| |
| pszReportPathName = MakeString(pszFormat, pszReportDirectory, pOptions->pszKey); |
| |
| fRptFile = fopen(pszReportPathName, "w"); |
| if (fRptFile == NULL) |
| mpp_err_msg(logWarn, progname, "Cannot open report file %s for writing. Will use stdout instead.\n", |
| pszReportPathName); |
| |
| /* buffer to store the complete restore report */ |
| reportBuf = createPQExpBuffer(); |
| |
| /* |
| * Write to buffer a report showing the timestamp key, what the command |
| * line options were, which segment databases were backed up, to where, |
| * and any errors that occurred. |
| */ |
| appendPQExpBuffer(reportBuf, "\nGreenplum Database Restore Report\n"); |
| appendPQExpBuffer(reportBuf, "Timestamp Key: %s\n", pOptions->pszKey); |
| |
| appendPQExpBuffer(reportBuf, "gp_restore Command Line: %s\n", |
| StringNotNull(pOptions->pszCmdLineParms, "None")); |
| |
| appendPQExpBuffer(reportBuf, "Pass through Command Line Options: %s\n", |
| StringNotNull(pOptions->pszPassThroughParms, "None")); |
| |
| appendPQExpBuffer(reportBuf, "Compression Program: %s\n", |
| StringNotNull(pOptions->pszCompressionProgram, "None")); |
| |
| appendPQExpBuffer(reportBuf, "\n"); |
| appendPQExpBuffer(reportBuf, "Individual Results\n"); |
| |
| |
| failCount = 1; |
| |
| pParm = pMasterParm; |
| |
| pszStatus = "Failed with error: \n{"; |
| if (localMsg != NULL) |
| pszMsg = (char *) localMsg; |
| else |
| pszMsg = pParm->pszErrorMsg; |
| if (pszMsg == NULL) |
| pszMsg = ""; |
| |
| appendPQExpBuffer(reportBuf, "\tRestore of database \"%s\" on Master database: %s%s%s\n", |
| StringNotNull(pOptions->pszDBName, "UNSPECIFIED"), |
| pszStatus, |
| pszMsg, |
| (strcmp(pszMsg, "") == 0) ? "" : "}" |
| ); |
| |
| appendPQExpBuffer(reportBuf, "\n%s utility finished unsuccessfully with 1 failure.\n", progname); |
| |
| /* write report to report file */ |
| if (fRptFile != NULL) |
| { |
| mpp_msg(logInfo, progname, "Report results also written to %s.\n", pszReportPathName); |
| fprintf(fRptFile, "%s", reportBuf->data); |
| fclose(fRptFile); |
| } |
| |
| /* write report to stdout */ |
| fprintf(stdout, "%s", reportBuf->data); |
| |
| if (pszReportPathName != NULL) |
| free(pszReportPathName); |
| |
| destroyPQExpBuffer(reportBuf); |
| |
| return failCount; |
| } |
| |