| /* |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, |
| software distributed under the License is distributed on an |
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| KIND, either express or implied. See the License for the |
| specific language governing permissions and limitations |
| under the License. |
| */ |
| |
| package cmd |
| |
| import ( |
| "database/sql" |
| "strconv" |
| |
| "github.com/apache/cloudberry-go-libs/gplog" |
| "github.com/spf13/cobra" |
| "github.com/spf13/pflag" |
| |
| "github.com/apache/cloudberry-backup/gpbackman/gpbckpconfig" |
| "github.com/apache/cloudberry-backup/gpbackman/textmsg" |
| "github.com/apache/cloudberry-backup/utils" |
| ) |
| |
| // Flags for the gpbackman backup-clean command (backupCleanCmd) |
| var ( |
| backupCleanBeforeTimestamp string |
| backupCleanAfterTimestamp string |
| backupCleanPluginConfigFile string |
| backupCleanBackupDir string |
| backupCleanOlderThanDays uint |
| backupCleanParallelProcesses int |
| backupCleanCascade bool |
| backupCleanNoHistorySyncStandby bool |
| backupCleanDatabase string |
| ) |
| |
| var backupCleanCmd = &cobra.Command{ |
| Use: "backup-clean", |
| Short: "Delete all existing backups older than the specified time condition", |
| Long: `Delete all existing backups older than the specified time condition. |
| |
| To delete backup sets older than the given timestamp, use the --before-timestamp option. |
| To delete backup sets older than the given number of days, use the --older-than-day option. |
| To delete backup sets newer than the given timestamp, use the --after-timestamp option. |
| Only --older-than-days, --before-timestamp or --after-timestamp option must be specified. |
| |
| Use --database to clean backups only for the specified database. Without --database, |
| cleanup includes backups for all databases in the history database. |
| Database names are matched exactly and case-sensitively against backup history. |
| For database names that require quoting, include the double quotes in the --database value. |
| |
| By default, the existence of dependent backups is checked and deletion process is not performed, |
| unless the --cascade option is passed in. |
| |
| By default, the deletion will be performed for local backup. |
| |
| The full path to the backup directory can be set using the --backup-dir option. |
| |
| For local backups the following logic are applied: |
| * If the --backup-dir option is specified, the deletion will be performed in provided path. |
| * If the --backup-dir option is not specified, but the backup was made with --backup-dir flag for gpbackup, the deletion will be performed in the backup manifest path. |
| * If the --backup-dir option is not specified and backup directory is not specified in backup manifest, the deletion will be performed in backup folder in the master and segments data directories. |
| * If backup is not local, the error will be returned. |
| |
| For control over the number of parallel processes and ssh connections to delete local backups, the --parallel-processes option can be used. |
| |
| The storage plugin config file location can be set using the --plugin-config option. |
| The full path to the file is required. In this case, the deletion will be performed using the storage plugin. |
| |
| For non local backups the following logic are applied: |
| * If the --plugin-config option is specified, the deletion will be performed using the storage plugin. |
| * If backup is local, the error will be returned. |
| |
| The gpbackup_history.db file location can be set using the --history-db option. |
| Can be specified only once. The full path to the file is required. |
| If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag.`, |
| Args: cobra.NoArgs, |
| Run: func(cmd *cobra.Command, args []string) { |
| doRootFlagValidation(cmd.Flags(), checkFileExistsConst) |
| doCleanBackupFlagValidation(cmd.Flags()) |
| doCleanBackup() |
| }, |
| } |
| |
| func init() { |
| rootCmd.AddCommand(backupCleanCmd) |
| backupCleanCmd.Flags().StringVar( |
| &backupCleanDatabase, |
| databaseFlagName, |
| "", |
| "delete backup sets only for the specified database", |
| ) |
| backupCleanCmd.PersistentFlags().StringVar( |
| &backupCleanPluginConfigFile, |
| pluginConfigFileFlagName, |
| "", |
| "the full path to plugin config file", |
| ) |
| backupCleanCmd.PersistentFlags().BoolVar( |
| &backupCleanCascade, |
| cascadeFlagName, |
| false, |
| "delete all dependent backups", |
| ) |
| backupCleanCmd.PersistentFlags().UintVar( |
| &backupCleanOlderThanDays, |
| olderThanDaysFlagName, |
| 0, |
| "delete backup sets older than the given number of days", |
| ) |
| backupCleanCmd.PersistentFlags().StringVar( |
| &backupCleanBeforeTimestamp, |
| beforeTimestampFlagName, |
| "", |
| "delete backup sets older than the given timestamp", |
| ) |
| backupCleanCmd.PersistentFlags().StringVar( |
| &backupCleanAfterTimestamp, |
| afterTimestampFlagName, |
| "", |
| "delete backup sets newer than the given timestamp", |
| ) |
| backupCleanCmd.PersistentFlags().StringVar( |
| &backupCleanBackupDir, |
| backupDirFlagName, |
| "", |
| "the full path to backup directory for local backups", |
| ) |
| backupCleanCmd.PersistentFlags().IntVar( |
| &backupCleanParallelProcesses, |
| parallelProcessesFlagName, |
| 1, |
| "the number of parallel processes to delete local backups", |
| ) |
| backupCleanCmd.Flags().BoolVar( |
| &backupCleanNoHistorySyncStandby, |
| noHistorySyncStandbyFlagName, |
| false, |
| "skip automatic gpbackup_history.db sync to standby coordinator after this command", |
| ) |
| backupCleanCmd.Flags().IntVar( |
| &historyStandbySyncTimeoutSeconds, |
| historySyncStandbyTimeoutFlagName, |
| historySyncStandbyTimeoutDefault, |
| "shared rsync and remote install timeout in seconds; must be an integer between 1 and 86400", |
| ) |
| backupCleanCmd.MarkFlagsMutuallyExclusive(beforeTimestampFlagName, olderThanDaysFlagName, afterTimestampFlagName) |
| } |
| |
| // These flag checks are applied only for backup-clean command. |
| func doCleanBackupFlagValidation(flags *pflag.FlagSet) { |
| var err error |
| if flags.Changed(databaseFlagName) && backupCleanDatabase == "" { |
| gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanDatabase, databaseFlagName, textmsg.ErrorEmptyDatabase())) |
| execOSExit(exitErrorCode) |
| } |
| // If before-timestamp flag is specified and have correct values. |
| if flags.Changed(beforeTimestampFlagName) { |
| err = gpbckpconfig.CheckTimestamp(backupCleanBeforeTimestamp) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanBeforeTimestamp, beforeTimestampFlagName, err)) |
| execOSExit(exitErrorCode) |
| } |
| beforeTimestamp = backupCleanBeforeTimestamp |
| } |
| if flags.Changed(olderThanDaysFlagName) { |
| beforeTimestamp = gpbckpconfig.GetTimestampOlderThan(backupCleanOlderThanDays) |
| } |
| // If after-timestamp flag is specified and have correct values. |
| if flags.Changed(afterTimestampFlagName) { |
| err = gpbckpconfig.CheckTimestamp(backupCleanAfterTimestamp) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanAfterTimestamp, afterTimestampFlagName, err)) |
| execOSExit(exitErrorCode) |
| } |
| afterTimestamp = backupCleanAfterTimestamp |
| } |
| // backup-dir anf plugin-config flags cannot be used together. |
| err = checkCompatibleFlags(flags, backupDirFlagName, pluginConfigFileFlagName) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, backupDirFlagName, pluginConfigFileFlagName)) |
| execOSExit(exitErrorCode) |
| } |
| // If parallel-processes flag is specified and have correct values. |
| if flags.Changed(parallelProcessesFlagName) && !gpbckpconfig.IsPositiveValue(backupCleanParallelProcesses) { |
| gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(strconv.Itoa(backupCleanParallelProcesses), parallelProcessesFlagName, err)) |
| execOSExit(exitErrorCode) |
| } |
| // plugin-config and parallel-precesses flags cannot be used together. |
| err = checkCompatibleFlags(flags, parallelProcessesFlagName, pluginConfigFileFlagName) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, parallelProcessesFlagName, pluginConfigFileFlagName)) |
| execOSExit(exitErrorCode) |
| } |
| // If backup-dir flag is specified and it exists and the full path is specified. |
| if flags.Changed(backupDirFlagName) { |
| err = gpbckpconfig.CheckFullPath(backupCleanBackupDir, checkFileExistsConst) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanBackupDir, backupDirFlagName, err)) |
| execOSExit(exitErrorCode) |
| } |
| } |
| // If plugin-config flag is specified and it exists and the full path is specified. |
| if flags.Changed(pluginConfigFileFlagName) { |
| err = gpbckpconfig.CheckFullPath(backupCleanPluginConfigFile, checkFileExistsConst) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanPluginConfigFile, pluginConfigFileFlagName, err)) |
| execOSExit(exitErrorCode) |
| } |
| } |
| if beforeTimestamp == "" && afterTimestamp == "" { |
| gplog.Error("%s", textmsg.ErrorTextUnableValidateValue(textmsg.ErrorValidationValue(), olderThanDaysFlagName, beforeTimestampFlagName, afterTimestampFlagName)) |
| execOSExit(exitErrorCode) |
| } |
| } |
| |
| func doCleanBackup() { |
| logHeadersDebug() |
| runHistoryMutationWithStandbySync(cleanBackup, backupCleanNoHistorySyncStandby) |
| } |
| |
| func cleanBackup() error { |
| hDB, err := gpbckpconfig.OpenHistoryDB(getHistoryDBPath(rootHistoryDB, rootAutoLoadHistoryDB)) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("open", err)) |
| return err |
| } |
| defer func() { |
| closeErr := hDB.Close() |
| if closeErr != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("close", closeErr)) |
| } |
| }() |
| if backupCleanPluginConfigFile != "" { |
| pluginConfig, err := utils.ReadPluginConfig(backupCleanPluginConfigFile) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableReadPluginConfigFile(err)) |
| return err |
| } |
| err = backupCleanDBPlugin(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanDatabase, backupCleanPluginConfigFile, pluginConfig, hDB) |
| if err != nil { |
| return err |
| } |
| } else { |
| err := backupCleanDBLocal(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanDatabase, backupCleanBackupDir, backupCleanParallelProcesses, hDB) |
| if err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, databaseName, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB) error { |
| backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, databaseName, hDB) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) |
| return err |
| } |
| if len(backupList) > 0 { |
| gplog.Debug("%s", textmsg.InfoTextBackupDeleteList(backupList)) |
| // Execute deletion for each backup. |
| // Use backupDeleteDBPlugin function from backup-delete command. |
| // Don't use force deletes and ignore errors for mass deletion. |
| err = backupDeleteDBPlugin(backupList, deleteCascade, false, false, pluginConfigPath, pluginConfig, hDB) |
| if err != nil { |
| return err |
| } |
| } else { |
| gplog.Info("%s", textmsg.InfoTextNothingToDo()) |
| } |
| return nil |
| } |
| |
| func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, databaseName, backupDir string, maxParallelProcesses int, hDB *sql.DB) error { |
| backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, databaseName, hDB) |
| if err != nil { |
| gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) |
| return err |
| } |
| if len(backupList) > 0 { |
| gplog.Debug("%s", textmsg.InfoTextBackupDeleteList(backupList)) |
| err = backupDeleteDBLocal(backupList, backupDir, deleteCascade, false, false, maxParallelProcesses, hDB) |
| if err != nil { |
| return err |
| } |
| } else { |
| gplog.Info("%s", textmsg.InfoTextNothingToDo()) |
| } |
| return nil |
| } |
| |
| // Get the list of backup names for deletion. |
| func fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, databaseName string, hDB *sql.DB) ([]string, error) { |
| var backupList []string |
| var err error |
| if cutOffTimestamp != "" { |
| backupList, err = gpbckpconfig.GetBackupNamesBeforeTimestamp(cutOffTimestamp, databaseName, hDB) |
| if err != nil { |
| return nil, err |
| } |
| } |
| if cutOffAfterTimestamp != "" { |
| backupList, err = gpbckpconfig.GetBackupNamesAfterTimestamp(cutOffAfterTimestamp, databaseName, hDB) |
| if err != nil { |
| return nil, err |
| } |
| } |
| return backupList, nil |
| } |