| package backup |
| |
| import ( |
| "fmt" |
| "os" |
| "path" |
| "reflect" |
| |
| "github.com/apache/cloudberry-backup/history" |
| "github.com/apache/cloudberry-backup/options" |
| "github.com/apache/cloudberry-backup/report" |
| "github.com/apache/cloudberry-backup/toc" |
| "github.com/apache/cloudberry-backup/utils" |
| "github.com/apache/cloudberry-go-libs/cluster" |
| "github.com/apache/cloudberry-go-libs/dbconn" |
| "github.com/apache/cloudberry-go-libs/gplog" |
| "github.com/nightlyone/lockfile" |
| "github.com/pkg/errors" |
| ) |
| |
| /* |
| * This file contains wrapper functions that group together functions relating |
| * to querying and printing metadata, so that the logic for each object type |
| * can all be in one place and backup.go can serve as a high-level look at the |
| * overall backup flow. |
| */ |
| |
| /* |
| * Setup and validation wrapper functions |
| */ |
| |
| func SetLoggerVerbosity() { |
| gplog.SetLogFileVerbosity(gplog.LOGINFO) |
| if MustGetFlagBool(options.QUIET) { |
| gplog.SetVerbosity(gplog.LOGERROR) |
| gplog.SetLogFileVerbosity(gplog.LOGERROR) |
| } else if MustGetFlagBool(options.DEBUG) { |
| gplog.SetVerbosity(gplog.LOGDEBUG) |
| gplog.SetLogFileVerbosity(gplog.LOGDEBUG) |
| } else if MustGetFlagBool(options.VERBOSE) { |
| gplog.SetVerbosity(gplog.LOGVERBOSE) |
| gplog.SetLogFileVerbosity(gplog.LOGVERBOSE) |
| } |
| } |
| |
| func initializeConnectionPool(timestamp string) { |
| var numConns int |
| var err error |
| connectionPool = dbconn.NewDBConnFromEnvironment(MustGetFlagString(options.DBNAME)) |
| /* |
| * The additional connection is used to hold the locks and process any deferred |
| * tables while not being assigned any tables up front. The benefit is that the |
| * connection holding the locks won't be overburdened by processing an assigned |
| * set of tables and any extra deferred tables, at the cost of making one |
| * additional connection to the database. |
| */ |
| switch true { |
| case FlagChanged(options.COPY_QUEUE_SIZE): |
| numConns = MustGetFlagInt(options.COPY_QUEUE_SIZE) + 1 |
| case FlagChanged(options.JOBS): |
| numConns = MustGetFlagInt(options.JOBS) + 1 |
| default: |
| numConns = 2 |
| } |
| |
| gplog.Verbose("Initializing %d database connections", numConns) |
| connectionPool.MustConnect(numConns) |
| |
| utils.ValidateGPDBVersionCompatibility(connectionPool) |
| InitializeMetadataParams(connectionPool) |
| // Begin transactions, initialize the synchronized snapshot, and set session GUCs |
| for connNum := 0; connNum < connectionPool.NumConns; connNum++ { |
| connectionPool.MustExec(fmt.Sprintf("SET application_name TO 'gpbackup_%s'", timestamp), connNum) |
| // Begins transaction in repeatable read |
| connectionPool.MustBegin(connNum) |
| if ((connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast(SNAPSHOT_GPDB_MIN_VERSION)) || connectionPool.Version.IsCBDB()) && backupSnapshot == "" { |
| backupSnapshot, err = GetSynchronizedSnapshot(connectionPool) |
| if err != nil { |
| gplog.FatalOnError(err) |
| } |
| } |
| |
| SetSessionGUCs(connNum) |
| } |
| } |
| |
| func SetSessionGUCs(connNum int) { |
| // These GUCs ensure the dumps portability accross systems |
| connectionPool.MustExec("SET search_path TO pg_catalog", connNum) |
| connectionPool.MustExec("SET statement_timeout = 0", connNum) |
| connectionPool.MustExec("SET DATESTYLE = ISO", connNum) |
| connectionPool.MustExec("SET enable_mergejoin TO off", connNum) |
| |
| // To avoid any complicated version diffs of setting this GUC, |
| // we use set_config() with a subquery getting the max value. |
| connectionPool.MustExec("SELECT set_config('extra_float_digits', (SELECT max_val FROM pg_settings WHERE name = 'extra_float_digits'), false)", connNum) |
| |
| connectionPool.MustExec("SET synchronize_seqscans TO off", connNum) |
| |
| if (connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast("6")) || connectionPool.Version.IsCBDB() { |
| connectionPool.MustExec("SET INTERVALSTYLE = POSTGRES", connNum) |
| connectionPool.MustExec("SET lock_timeout = 0", connNum) |
| } |
| |
| if (connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast("7")) || connectionPool.Version.IsCBDB() { |
| // This is a GPDB7+ GUC that can terminate sessions with open transactions that have been idle for too long, so we disable it. |
| connectionPool.MustExec("SET idle_in_transaction_session_timeout = 0", connNum) |
| } |
| } |
| |
| func NewBackupConfig(dbName string, dbVersion string, backupVersion string, plugin string, timestamp string, opts options.Options) *history.BackupConfig { |
| backupConfig := history.BackupConfig{ |
| BackupDir: MustGetFlagString(options.BACKUP_DIR), |
| BackupVersion: backupVersion, |
| Compressed: !MustGetFlagBool(options.NO_COMPRESSION), |
| CompressionType: MustGetFlagString(options.COMPRESSION_TYPE), |
| DatabaseName: dbName, |
| DatabaseVersion: dbVersion, |
| DataOnly: MustGetFlagBool(options.DATA_ONLY), |
| ExcludeRelations: MustGetFlagStringArray(options.EXCLUDE_RELATION), |
| ExcludeSchemaFiltered: len(MustGetFlagStringArray(options.EXCLUDE_SCHEMA)) > 0, |
| ExcludeSchemas: MustGetFlagStringArray(options.EXCLUDE_SCHEMA), |
| ExcludeTableFiltered: len(MustGetFlagStringArray(options.EXCLUDE_RELATION)) > 0, |
| IncludeRelations: opts.GetOriginalIncludedTables(), |
| IncludeSchemaFiltered: len(MustGetFlagStringArray(options.INCLUDE_SCHEMA)) > 0, |
| IncludeSchemas: MustGetFlagStringArray(options.INCLUDE_SCHEMA), |
| IncludeTableFiltered: len(opts.GetOriginalIncludedTables()) > 0, |
| Incremental: MustGetFlagBool(options.INCREMENTAL), |
| LeafPartitionData: MustGetFlagBool(options.LEAF_PARTITION_DATA), |
| MetadataOnly: MustGetFlagBool(options.METADATA_ONLY), |
| Plugin: plugin, |
| SingleDataFile: MustGetFlagBool(options.SINGLE_DATA_FILE), |
| Timestamp: timestamp, |
| WithoutGlobals: MustGetFlagBool(options.WITHOUT_GLOBALS), |
| WithStatistics: MustGetFlagBool(options.WITH_STATS), |
| Status: history.BackupStatusInProgress, |
| } |
| |
| return &backupConfig |
| } |
| |
| func initializeBackupReport(opts options.Options) { |
| escapedDBName := dbconn.MustSelectString(connectionPool, fmt.Sprintf("select quote_ident(datname) AS string FROM pg_database where datname='%s'", utils.EscapeSingleQuotes(connectionPool.DBName))) |
| plugin := "" |
| if pluginConfig != nil { |
| _, plugin = path.Split(pluginConfig.ExecutablePath) |
| } |
| config := NewBackupConfig(escapedDBName, connectionPool.Version.VersionString, version, |
| plugin, globalFPInfo.Timestamp, opts) |
| |
| isFilteredBackup := config.IncludeTableFiltered || config.IncludeSchemaFiltered || |
| config.ExcludeTableFiltered || config.ExcludeSchemaFiltered |
| dbSize := "" |
| if !MustGetFlagBool(options.METADATA_ONLY) && !isFilteredBackup { |
| gplog.Verbose("Getting database size") |
| //Potentially expensive query |
| dbSize = GetDBSize(connectionPool) |
| } |
| |
| config.SegmentCount = len(globalCluster.ContentIDs) - 1 |
| |
| backupReport = &report.Report{ |
| DatabaseSize: dbSize, |
| BackupConfig: *config, |
| } |
| backupReport.ConstructBackupParamsString() |
| } |
| |
| func createBackupLockFile(timestamp string) { |
| var err error |
| var timestampLockFile string |
| metadataOnly := MustGetFlagBool(options.METADATA_ONLY) |
| backupDir := MustGetFlagString(options.BACKUP_DIR) |
| noHistory := MustGetFlagBool(options.NO_HISTORY) |
| if metadataOnly && noHistory && backupDir != "" { |
| err = os.MkdirAll(backupDir, 0777) |
| gplog.FatalOnError(err) |
| timestampLockFile = fmt.Sprintf("%s/%s.lck", backupDir, timestamp) |
| } else { |
| timestampLockFile = fmt.Sprintf("/tmp/%s.lck", timestamp) |
| } |
| backupLockFile, err = lockfile.New(timestampLockFile) |
| gplog.FatalOnError(err) |
| err = backupLockFile.TryLock() |
| if err != nil { |
| gplog.Error("%s", err.Error()) |
| gplog.Fatal(errors.Errorf("A backup with timestamp %s is already in progress. Wait 1 second and try the backup again.", timestamp), "") |
| } |
| } |
| |
| func createBackupDirectoriesOnAllHosts() { |
| remoteOutput := globalCluster.GenerateAndExecuteCommand("Creating backup directories", |
| cluster.ON_SEGMENTS|cluster.INCLUDE_COORDINATOR, |
| func(contentID int) string { |
| return fmt.Sprintf("mkdir -p %s", globalFPInfo.GetDirForContent(contentID)) |
| }) |
| globalCluster.CheckClusterError(remoteOutput, "Unable to create backup directories", func(contentID int) string { |
| return fmt.Sprintf("Unable to create backup directory %s", globalFPInfo.GetDirForContent(contentID)) |
| }) |
| } |
| |
| /* |
| * Metadata retrieval wrapper functions |
| */ |
| |
| func RetrieveAndProcessTables() ([]Table, []Table) { |
| includedRelations := GetIncludedUserTableRelations(connectionPool, IncludedRelationFqns) |
| tableRelations := ConvertRelationsOptionsToBackup(includedRelations) |
| |
| LockTables(connectionPool, tableRelations) |
| |
| if (connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast("6")) || connectionPool.Version.IsCBDB() { |
| tableRelations = append(tableRelations, GetForeignTableRelations(connectionPool)...) |
| } |
| |
| tables := ConstructDefinitionsForTables(connectionPool, tableRelations) |
| |
| metadataTables, dataTables := SplitTablesByPartitionType(tables, IncludedRelationFqns) |
| objectCounts["Tables"] = len(metadataTables) |
| |
| return metadataTables, dataTables |
| } |
| |
| func retrieveFunctions(sortables *[]Sortable, metadataMap MetadataMap) ([]Function, map[uint32]FunctionInfo) { |
| gplog.Verbose("Retrieving function information") |
| functionMetadata := GetMetadataForObjectType(connectionPool, TYPE_FUNCTION) |
| addToMetadataMap(functionMetadata, metadataMap) |
| functions := GetFunctions(connectionPool) |
| funcInfoMap := GetFunctionOidToInfoMap(connectionPool) |
| objectCounts["Functions"] = len(functions) |
| *sortables = append(*sortables, convertToSortableSlice(functions)...) |
| |
| return functions, funcInfoMap |
| } |
| |
| func retrieveTransforms(sortables *[]Sortable) { |
| if connectionPool.Version.IsGPDB() && connectionPool.Version.Before("7") { |
| return |
| } |
| gplog.Verbose("Retrieving transform information") |
| transforms := GetTransforms(connectionPool) |
| objectCounts["Transforms"] = len(transforms) |
| *sortables = append(*sortables, convertToSortableSlice(transforms)...) |
| } |
| |
| func retrieveAndBackupTypes(metadataFile *utils.FileWithByteCount, sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving type information") |
| shells := GetShellTypes(connectionPool) |
| bases := GetBaseTypes(connectionPool) |
| composites := GetCompositeTypes(connectionPool) |
| domains := GetDomainTypes(connectionPool) |
| rangeTypes := make([]RangeType, 0) |
| if (connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast("6")) || connectionPool.Version.IsCBDB() { |
| rangeTypes = GetRangeTypes(connectionPool) |
| } |
| typeMetadata := GetMetadataForObjectType(connectionPool, TYPE_TYPE) |
| |
| backupShellTypes(metadataFile, shells, bases, rangeTypes) |
| backupEnumTypes(metadataFile, typeMetadata) |
| |
| objectCounts["Types"] += len(shells) |
| objectCounts["Types"] += len(bases) |
| objectCounts["Types"] += len(composites) |
| objectCounts["Types"] += len(domains) |
| objectCounts["Types"] += len(rangeTypes) |
| *sortables = append(*sortables, convertToSortableSlice(bases)...) |
| *sortables = append(*sortables, convertToSortableSlice(composites)...) |
| *sortables = append(*sortables, convertToSortableSlice(domains)...) |
| *sortables = append(*sortables, convertToSortableSlice(rangeTypes)...) |
| addToMetadataMap(typeMetadata, metadataMap) |
| } |
| |
| func retrieveConstraints(sortables *[]Sortable, metadataMap MetadataMap, tables ...Relation) ([]Constraint, []Constraint, MetadataMap) { |
| gplog.Verbose("Retrieving constraints") |
| constraints := GetConstraints(connectionPool, tables...) |
| if len(constraints) > 0 && ((connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast("7")) || |
| connectionPool.Version.IsCBDB()) { |
| RenameExchangedPartitionConstraints(connectionPool, &constraints) |
| } |
| |
| //split into domain constraints and all others, as they are handled differently downstream |
| domainConstraints := make([]Constraint, 0) |
| nonDomainConstraints := make([]Constraint, 0) |
| for _, con := range constraints { |
| if con.IsDomainConstraint { |
| domainConstraints = append(domainConstraints, con) |
| } else { |
| nonDomainConstraints = append(nonDomainConstraints, con) |
| } |
| } |
| objectCounts["Constraints"] = len(nonDomainConstraints) |
| conMetadata := GetCommentsForObjectType(connectionPool, TYPE_CONSTRAINT) |
| *sortables = append(*sortables, convertToSortableSlice(nonDomainConstraints)...) |
| addToMetadataMap(conMetadata, metadataMap) |
| return domainConstraints, nonDomainConstraints, conMetadata |
| } |
| |
| func retrieveAndBackupSequences(metadataFile *utils.FileWithByteCount, |
| relationMetadata MetadataMap) []Sequence { |
| gplog.Verbose("Writing CREATE SEQUENCE statements to metadata file") |
| sequences := GetAllSequences(connectionPool) |
| objectCounts["Sequences"] = len(sequences) |
| PrintCreateSequenceStatements(metadataFile, globalTOC, sequences, relationMetadata) |
| return sequences |
| } |
| |
| func retrieveProtocols(sortables *[]Sortable, metadataMap MetadataMap) []ExternalProtocol { |
| gplog.Verbose("Retrieving protocols") |
| protocols := GetExternalProtocols(connectionPool) |
| objectCounts["Protocols"] = len(protocols) |
| protoMetadata := GetMetadataForObjectType(connectionPool, TYPE_PROTOCOL) |
| |
| *sortables = append(*sortables, convertToSortableSlice(protocols)...) |
| addToMetadataMap(protoMetadata, metadataMap) |
| |
| return protocols |
| } |
| |
| func retrieveViews(sortables *[]Sortable) { |
| gplog.Verbose("Retrieving views") |
| views := GetAllViews(connectionPool) |
| objectCounts["Views"] = len(views) |
| |
| *sortables = append(*sortables, convertToSortableSlice(views)...) |
| } |
| |
| func retrieveTSObjects(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving Text Search Parsers") |
| retrieveTSParsers(sortables, metadataMap) |
| retrieveTSConfigurations(sortables, metadataMap) |
| retrieveTSTemplates(sortables, metadataMap) |
| retrieveTSDictionaries(sortables, metadataMap) |
| } |
| |
| func retrieveTSParsers(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving Text Search Parsers") |
| parsers := GetTextSearchParsers(connectionPool) |
| objectCounts["Text Search Parsers"] = len(parsers) |
| parserMetadata := GetCommentsForObjectType(connectionPool, TYPE_TS_PARSER) |
| |
| *sortables = append(*sortables, convertToSortableSlice(parsers)...) |
| addToMetadataMap(parserMetadata, metadataMap) |
| } |
| |
| func retrieveTSTemplates(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving TEXT SEARCH TEMPLATE information") |
| templates := GetTextSearchTemplates(connectionPool) |
| objectCounts["Text Search Templates"] = len(templates) |
| templateMetadata := GetCommentsForObjectType(connectionPool, TYPE_TS_TEMPLATE) |
| |
| *sortables = append(*sortables, convertToSortableSlice(templates)...) |
| addToMetadataMap(templateMetadata, metadataMap) |
| } |
| |
| func retrieveTSDictionaries(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving TEXT SEARCH DICTIONARY information") |
| dictionaries := GetTextSearchDictionaries(connectionPool) |
| objectCounts["Text Search Dictionaries"] = len(dictionaries) |
| dictionaryMetadata := GetMetadataForObjectType(connectionPool, TYPE_TS_DICTIONARY) |
| |
| *sortables = append(*sortables, convertToSortableSlice(dictionaries)...) |
| addToMetadataMap(dictionaryMetadata, metadataMap) |
| } |
| |
| func retrieveTSConfigurations(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving TEXT SEARCH CONFIGURATION information") |
| configurations := GetTextSearchConfigurations(connectionPool) |
| objectCounts["Text Search Configurations"] = len(configurations) |
| configurationMetadata := GetMetadataForObjectType(connectionPool, TYPE_TS_CONFIGURATION) |
| |
| *sortables = append(*sortables, convertToSortableSlice(configurations)...) |
| addToMetadataMap(configurationMetadata, metadataMap) |
| } |
| |
| func retrieveOperatorObjects(sortables *[]Sortable, metadataMap MetadataMap) { |
| retrieveOperators(sortables, metadataMap) |
| retrieveOperatorClasses(sortables, metadataMap) |
| } |
| |
| func retrieveOperators(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving OPERATOR information") |
| operators := GetOperators(connectionPool) |
| objectCounts["Operators"] = len(operators) |
| operatorMetadata := GetMetadataForObjectType(connectionPool, TYPE_OPERATOR) |
| |
| *sortables = append(*sortables, convertToSortableSlice(operators)...) |
| addToMetadataMap(operatorMetadata, metadataMap) |
| } |
| |
| func retrieveOperatorClasses(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving OPERATOR CLASS information") |
| operatorClasses := GetOperatorClasses(connectionPool) |
| objectCounts["Operator Classes"] = len(operatorClasses) |
| operatorClassMetadata := GetMetadataForObjectType(connectionPool, TYPE_OPERATOR_CLASS) |
| |
| *sortables = append(*sortables, convertToSortableSlice(operatorClasses)...) |
| addToMetadataMap(operatorClassMetadata, metadataMap) |
| } |
| |
| func retrieveAggregates(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving AGGREGATE information") |
| aggregates := GetAggregates(connectionPool) |
| objectCounts["Aggregates"] = len(aggregates) |
| aggMetadata := GetMetadataForObjectType(connectionPool, TYPE_AGGREGATE) |
| |
| *sortables = append(*sortables, convertToSortableSlice(aggregates)...) |
| addToMetadataMap(aggMetadata, metadataMap) |
| } |
| |
| func retrieveCasts(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Retrieving CAST information") |
| casts := GetCasts(connectionPool) |
| objectCounts["Casts"] = len(casts) |
| castMetadata := GetCommentsForObjectType(connectionPool, TYPE_CAST) |
| |
| *sortables = append(*sortables, convertToSortableSlice(casts)...) |
| addToMetadataMap(castMetadata, metadataMap) |
| } |
| |
| func retrieveFDWObjects(sortables *[]Sortable, metadataMap MetadataMap) { |
| if !(connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast("6")) || connectionPool.Version.IsCBDB() { |
| return |
| } |
| retrieveForeignDataWrappers(sortables, metadataMap) |
| retrieveForeignServers(sortables, metadataMap) |
| retrieveUserMappings(sortables) |
| } |
| |
| func retrieveForeignDataWrappers(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Writing CREATE FOREIGN DATA WRAPPER statements to metadata file") |
| wrappers := GetForeignDataWrappers(connectionPool) |
| objectCounts["Foreign Data Wrappers"] = len(wrappers) |
| fdwMetadata := GetMetadataForObjectType(connectionPool, TYPE_FOREIGN_DATA_WRAPPER) |
| |
| *sortables = append(*sortables, convertToSortableSlice(wrappers)...) |
| addToMetadataMap(fdwMetadata, metadataMap) |
| } |
| |
| func retrieveForeignServers(sortables *[]Sortable, metadataMap MetadataMap) { |
| gplog.Verbose("Writing CREATE SERVER statements to metadata file") |
| servers := GetForeignServers(connectionPool) |
| objectCounts["Foreign Servers"] = len(servers) |
| serverMetadata := GetMetadataForObjectType(connectionPool, TYPE_FOREIGN_SERVER) |
| |
| *sortables = append(*sortables, convertToSortableSlice(servers)...) |
| addToMetadataMap(serverMetadata, metadataMap) |
| } |
| |
| func retrieveUserMappings(sortables *[]Sortable) { |
| gplog.Verbose("Writing CREATE USER MAPPING statements to metadata file") |
| mappings := GetUserMappings(connectionPool) |
| objectCounts["User Mappings"] = len(mappings) |
| // No comments, owners, or ACLs on UserMappings so no need to get metadata |
| |
| *sortables = append(*sortables, convertToSortableSlice(mappings)...) |
| } |
| |
| func backupSessionGUC(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing Session Configuration Parameters to metadata file") |
| gucs := GetSessionGUCs(connectionPool) |
| PrintSessionGUCs(metadataFile, globalTOC, gucs) |
| } |
| |
| /* |
| * Global metadata wrapper functions |
| */ |
| |
| func backupTablespaces(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE TABLESPACE statements to metadata file") |
| tablespaces := GetTablespaces(connectionPool) |
| objectCounts["Tablespaces"] = len(tablespaces) |
| tablespaceMetadata := GetMetadataForObjectType(connectionPool, TYPE_TABLESPACE) |
| PrintCreateTablespaceStatements(metadataFile, globalTOC, tablespaces, tablespaceMetadata) |
| } |
| |
| func backupCreateDatabase(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE DATABASE statement to metadata file") |
| defaultDB := GetDefaultDatabaseEncodingInfo(connectionPool) |
| db := GetDatabaseInfo(connectionPool) |
| dbMetadata := GetMetadataForObjectType(connectionPool, TYPE_DATABASE) |
| PrintCreateDatabaseStatement(metadataFile, globalTOC, defaultDB, db, dbMetadata) |
| } |
| |
| func backupDatabaseGUCs(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing Database Configuration Parameters to metadata file") |
| databaseGucs := GetDatabaseGUCs(connectionPool) |
| objectCounts["Database GUCs"] = len(databaseGucs) |
| PrintDatabaseGUCs(metadataFile, globalTOC, databaseGucs, connectionPool.DBName) |
| } |
| |
| func backupResourceQueues(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE RESOURCE QUEUE statements to metadata file") |
| resQueues := GetResourceQueues(connectionPool) |
| objectCounts["Resource Queues"] = len(resQueues) |
| resQueueMetadata := GetCommentsForObjectType(connectionPool, TYPE_RESOURCE_QUEUE) |
| PrintCreateResourceQueueStatements(metadataFile, globalTOC, resQueues, resQueueMetadata) |
| } |
| |
| func backupResourceGroups(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE RESOURCE GROUP statements to metadata file") |
| if connectionPool.Version.IsGPDB() && connectionPool.Version.Before("7") { |
| resGroups := GetResourceGroups[ResourceGroupBefore7](connectionPool) |
| objectCounts["Resource Groups"] = len(resGroups) |
| resGroupMetadata := GetCommentsForObjectType(connectionPool, TYPE_RESOURCE_GROUP) |
| PrintResetResourceGroupStatements(metadataFile, globalTOC) |
| PrintCreateResourceGroupStatementsBefore7(metadataFile, globalTOC, resGroups, resGroupMetadata) |
| } else { // GPDB7+ |
| resGroups := GetResourceGroups[ResourceGroupAtLeast7](connectionPool) |
| objectCounts["Resource Groups"] = len(resGroups) |
| resGroupMetadata := GetCommentsForObjectType(connectionPool, TYPE_RESOURCE_GROUP) |
| PrintResetResourceGroupStatements(metadataFile, globalTOC) |
| PrintCreateResourceGroupStatementsAtLeast7(metadataFile, globalTOC, resGroups, resGroupMetadata) |
| } |
| } |
| |
| func backupRoles(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE ROLE statements to metadata file") |
| roles := GetRoles(connectionPool) |
| objectCounts["Roles"] = len(roles) |
| roleMetadata := GetMetadataForObjectType(connectionPool, TYPE_ROLE) |
| PrintCreateRoleStatements(metadataFile, globalTOC, roles, roleMetadata) |
| } |
| |
| func backupRoleGUCs(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing ROLE Configuration Parameter to meadata file") |
| roleGUCs := GetRoleGUCs(connectionPool) |
| PrintRoleGUCStatements(metadataFile, globalTOC, roleGUCs) |
| } |
| |
| func backupRoleGrants(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing GRANT ROLE statements to metadata file") |
| roleMembers := GetRoleMembers(connectionPool) |
| PrintRoleMembershipStatements(metadataFile, globalTOC, roleMembers) |
| } |
| |
| /* |
| * Predata wrapper functions |
| */ |
| |
| func backupSchemas(metadataFile *utils.FileWithByteCount, partitionAlteredSchemas map[string]bool) { |
| gplog.Verbose("Writing CREATE SCHEMA statements to metadata file") |
| schemas := GetAllUserSchemas(connectionPool, partitionAlteredSchemas) |
| objectCounts["Schemas"] = len(schemas) |
| schemaMetadata := GetMetadataForObjectType(connectionPool, TYPE_SCHEMA) |
| PrintCreateSchemaStatements(metadataFile, globalTOC, schemas, schemaMetadata) |
| } |
| |
| func backupProceduralLanguages(metadataFile *utils.FileWithByteCount, |
| functions []Function, funcInfoMap map[uint32]FunctionInfo, functionMetadata MetadataMap) { |
| gplog.Verbose("Writing CREATE PROCEDURAL LANGUAGE statements to metadata file") |
| procLangs := GetProceduralLanguages(connectionPool) |
| objectCounts["Procedural Languages"] = len(procLangs) |
| langFuncs, _ := ExtractLanguageFunctions(functions, procLangs) |
| for _, langFunc := range langFuncs { |
| PrintCreateFunctionStatement(metadataFile, globalTOC, langFunc, functionMetadata[langFunc.GetUniqueID()]) |
| } |
| procLangMetadata := GetMetadataForObjectType(connectionPool, TYPE_PROC_LANGUAGE) |
| PrintCreateLanguageStatements(metadataFile, globalTOC, procLangs, funcInfoMap, procLangMetadata) |
| } |
| |
| func backupShellTypes(metadataFile *utils.FileWithByteCount, shellTypes []ShellType, baseTypes []BaseType, rangeTypes []RangeType) { |
| gplog.Verbose("Writing CREATE TYPE statements for shell types to metadata file") |
| PrintCreateShellTypeStatements(metadataFile, globalTOC, shellTypes, baseTypes, rangeTypes) |
| } |
| |
| func backupEnumTypes(metadataFile *utils.FileWithByteCount, typeMetadata MetadataMap) { |
| gplog.Verbose("Writing CREATE TYPE statements for enum types to metadata file") |
| enums := GetEnumTypes(connectionPool) |
| objectCounts["Types"] += len(enums) |
| PrintCreateEnumTypeStatements(metadataFile, globalTOC, enums, typeMetadata) |
| } |
| |
| func backupAccessMethods(metadataFile *utils.FileWithByteCount) { |
| if connectionPool.Version.IsGPDB() && connectionPool.Version.Before("7") { |
| return |
| } |
| gplog.Verbose("Writing CREATE ACCESS METHOD statements to metadata file") |
| accessMethods := GetAccessMethods(connectionPool) |
| objectCounts["Access Methods"] = len(accessMethods) |
| accessMethodsMetadata := GetMetadataForObjectType(connectionPool, TYPE_ACCESS_METHOD) |
| PrintAccessMethodStatements(metadataFile, globalTOC, accessMethods, accessMethodsMetadata) |
| } |
| |
| func createBackupSet(objSlice []Sortable) (backupSet map[UniqueID]bool) { |
| backupSet = make(map[UniqueID]bool) |
| for _, obj := range objSlice { |
| backupSet[obj.GetUniqueID()] = true |
| } |
| |
| return backupSet |
| } |
| |
| func convertToSortableSlice(objSlice interface{}) []Sortable { |
| sortableSlice := make([]Sortable, 0) |
| s := reflect.ValueOf(objSlice) |
| |
| ret := make([]interface{}, s.Len()) |
| |
| for i := 0; i < s.Len(); i++ { |
| ret[i] = s.Index(i).Interface() |
| } |
| |
| for _, obj := range ret { |
| newObj := obj.(Sortable) |
| sortableSlice = append(sortableSlice, newObj) |
| } |
| |
| return sortableSlice |
| } |
| |
| func addToMetadataMap(newMetadata MetadataMap, metadataMap MetadataMap) { |
| for k, v := range newMetadata { |
| metadataMap[k] = v |
| } |
| } |
| |
| // This function is fairly unwieldy, but there's not really a good way to break it down |
| func backupDependentObjects(metadataFile *utils.FileWithByteCount, tables []Table, |
| protocols []ExternalProtocol, filteredMetadata MetadataMap, domainConstraints []Constraint, |
| sortables []Sortable, sequences []Sequence, funcInfoMap map[uint32]FunctionInfo, tableOnly bool) []View { |
| var sortedSlice []Sortable |
| gplog.Verbose("Writing CREATE statements for dependent objects to metadata file") |
| |
| backupSet := createBackupSet(sortables) |
| relevantDeps := GetDependencies(connectionPool, backupSet, tables) |
| viewsDependingOnConstraints := MarkViewsDependingOnConstraints(sortables, relevantDeps) |
| sortedSlice, globalTierMap = TopologicalSort(sortables, relevantDeps) |
| |
| PrintDependentObjectStatements(metadataFile, globalTOC, sortedSlice, filteredMetadata, domainConstraints, funcInfoMap) |
| PrintIdentityColumns(metadataFile, globalTOC, sequences) |
| PrintAlterSequenceStatements(metadataFile, globalTOC, sequences) |
| extPartInfo, partInfoMap := GetExternalPartitionInfo(connectionPool) |
| if connectionPool.Version.Before("7") && len(extPartInfo) > 0 { |
| gplog.Verbose("Writing EXCHANGE PARTITION statements to metadata file") |
| PrintExchangeExternalPartitionStatements(metadataFile, globalTOC, extPartInfo, partInfoMap, tables) |
| } |
| return viewsDependingOnConstraints |
| } |
| |
| func backupConversions(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE CONVERSION statements to metadata file") |
| conversions := GetConversions(connectionPool) |
| objectCounts["Conversions"] = len(conversions) |
| convMetadata := GetMetadataForObjectType(connectionPool, TYPE_CONVERSION) |
| PrintCreateConversionStatements(metadataFile, globalTOC, conversions, convMetadata) |
| } |
| |
| func backupOperatorFamilies(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE OPERATOR FAMILY statements to metadata file") |
| operatorFamilies := GetOperatorFamilies(connectionPool) |
| objectCounts["Operator Families"] = len(operatorFamilies) |
| operatorFamilyMetadata := GetMetadataForObjectType(connectionPool, TYPE_OPERATOR_FAMILY) |
| PrintCreateOperatorFamilyStatements(metadataFile, globalTOC, operatorFamilies, operatorFamilyMetadata) |
| } |
| |
| func backupCollations(metadataFile *utils.FileWithByteCount) { |
| if !((connectionPool.Version.IsGPDB() && connectionPool.Version.AtLeast("6")) || connectionPool.Version.IsCBDB()) { |
| return |
| } |
| gplog.Verbose("Writing CREATE COLLATION statements to metadata file") |
| collations := GetCollations(connectionPool) |
| objectCounts["Collations"] = len(collations) |
| collationMetadata := GetMetadataForObjectType(connectionPool, TYPE_COLLATION) |
| PrintCreateCollationStatements(metadataFile, globalTOC, collations, collationMetadata) |
| } |
| |
| func backupExtensions(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE EXTENSION statements to metadata file") |
| extensions := GetExtensions(connectionPool) |
| objectCounts["Extensions"] = len(extensions) |
| extensionMetadata := GetCommentsForObjectType(connectionPool, TYPE_EXTENSION) |
| PrintCreateExtensionStatements(metadataFile, globalTOC, extensions, extensionMetadata) |
| } |
| |
| func backupConstraints(metadataFile *utils.FileWithByteCount, constraints []Constraint, conMetadata MetadataMap) { |
| gplog.Verbose("Writing ADD CONSTRAINT statements to metadata file") |
| objectCounts["Constraints"] = len(constraints) |
| PrintConstraintStatements(metadataFile, globalTOC, constraints, conMetadata) |
| } |
| |
| func backupViewsDependingOnConstraints(metadataFile *utils.FileWithByteCount, views []View) { |
| gplog.Verbose("Writing CREATE VIEW statements for views that depend on constraints to metadata file") |
| PrintCreatePostdataViewStatements(metadataFile, globalTOC, views) |
| } |
| |
| /* |
| * Postdata wrapper functions |
| */ |
| |
| func backupIndexes(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE INDEX statements to metadata file") |
| indexes := GetIndexes(connectionPool) |
| objectCounts["Indexes"] = len(indexes) |
| if objectCounts["Indexes"] > 0 && (connectionPool.Version.IsGPDB() && connectionPool.Version.Is("6")) { |
| // This bug is not addressed in versions prior to GPDB6 |
| // New partition exchange syntax in GPDB7+ obviates the need for this renaming |
| RenameExchangedPartitionIndexes(connectionPool, &indexes) |
| } |
| indexMetadata := GetCommentsForObjectType(connectionPool, TYPE_INDEX) |
| PrintCreateIndexStatements(metadataFile, globalTOC, indexes, indexMetadata) |
| } |
| |
| func backupRules(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE RULE statements to metadata file") |
| rules := GetRules(connectionPool) |
| objectCounts["Rules"] = len(rules) |
| ruleMetadata := GetCommentsForObjectType(connectionPool, TYPE_RULE) |
| PrintCreateRuleStatements(metadataFile, globalTOC, rules, ruleMetadata) |
| } |
| |
| func backupTriggers(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE TRIGGER statements to metadata file") |
| triggers := GetTriggers(connectionPool) |
| objectCounts["Triggers"] = len(triggers) |
| triggerMetadata := GetCommentsForObjectType(connectionPool, TYPE_TRIGGER) |
| PrintCreateTriggerStatements(metadataFile, globalTOC, triggers, triggerMetadata) |
| } |
| |
| func backupEventTriggers(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE EVENT TRIGGER statements to metadata file") |
| eventTriggers := GetEventTriggers(connectionPool) |
| objectCounts["Event Triggers"] = len(eventTriggers) |
| eventTriggerMetadata := GetMetadataForObjectType(connectionPool, TYPE_EVENT_TRIGGER) |
| PrintCreateEventTriggerStatements(metadataFile, globalTOC, eventTriggers, eventTriggerMetadata) |
| } |
| |
| func backupRowLevelSecurityPolicies(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE POLICY statements to metadata file") |
| policies := GetPolicies(connectionPool) |
| objectCounts["Policies"] = len(policies) |
| ruleMetadata := GetCommentsForObjectType(connectionPool, TYPE_RULE) |
| PrintCreatePolicyStatements(metadataFile, globalTOC, policies, ruleMetadata) |
| } |
| |
| func backupDefaultPrivileges(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing ALTER DEFAULT PRIVILEGES statements to metadata file") |
| defaultPrivileges := GetDefaultPrivileges(connectionPool) |
| objectCounts["DEFAULT PRIVILEGES"] = len(defaultPrivileges) |
| PrintDefaultPrivilegesStatements(metadataFile, globalTOC, defaultPrivileges) |
| } |
| |
| func backupExtendedStatistic(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE STATISTICS statements to metadata file (for extended statistics)") |
| statisticsExt := GetExtendedStatistics(connectionPool) |
| objectCounts[toc.OBJ_STATISTICS_EXT] = len(statisticsExt) |
| statisticExtMetadata := GetMetadataForObjectType(connectionPool, TYPE_STATISTIC_EXT) |
| PrintCreateExtendedStatistics(metadataFile, globalTOC, statisticsExt, statisticExtMetadata) |
| } |
| |
| /* |
| * Data wrapper functions |
| */ |
| |
| func backupTableStatistics(statisticsFile *utils.FileWithByteCount, tables []Table) { |
| attStats := GetAttributeStatistics(connectionPool, tables) |
| tupleStats := GetTupleStatistics(connectionPool, tables) |
| |
| backupSessionGUC(statisticsFile) |
| PrintStatisticsStatements(statisticsFile, globalTOC, tables, attStats, tupleStats) |
| } |
| |
| func backupIncrementalMetadata() { |
| aoTableEntries := GetAOIncrementalMetadata(connectionPool) |
| globalTOC.IncrementalMetadata.AO = aoTableEntries |
| } |
| |
| func backupStorageServers(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE SERVER statements to metadata file") |
| servers := GetStorageServers(connectionPool) |
| objectCounts[toc.OBJ_STORAGE_SERVER] = len(servers) |
| serverMetadata := GetMetadataForObjectType(connectionPool, TYPE_STORAGE_SERVER) |
| PrintCreateStorageServerStatements(metadataFile, globalTOC, servers, serverMetadata) |
| } |
| |
| func backupStorageUserMappings(metadataFile *utils.FileWithByteCount) { |
| gplog.Verbose("Writing CREATE STORAGE USER MAPPING statements to metadata file") |
| users := GetStorageUserMapping(connectionPool) |
| objectCounts[toc.OBJ_STORAGE_USER_MAPPING] = len(users) |
| |
| PrintCreateStorageUserMappingStatements(metadataFile, globalTOC, users) |
| } |
