| #!/usr/bin/env python |
| # |
| # svn-backup-dumps.py -- Create dumpfiles to backup a subversion repository. |
| # |
| # ==================================================================== |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # ==================================================================== |
| # |
| # This script creates dump files from a subversion repository. |
| # It is intended for use in cron jobs and post-commit hooks. |
| # |
| # The basic operation modes are: |
| # 1. Create a full dump (revisions 0 to HEAD). |
| # 2. Create incremental dumps containing at most N revisions. |
| # 3. Create incremental single revision dumps (for use in post-commit). |
| # 4. Create incremental dumps containing everything since last dump. |
| # |
| # All dump files are prefixed with the basename of the repository. All |
| # examples below assume that the repository '/srv/svn/repos/src' is |
| # dumped so all dumpfiles start with 'src'. |
| # |
| # Optional functionality: |
| # 5. Create gzipped dump files. |
| # 6. Create bzipped dump files. |
| # 7. Transfer the dumpfile to another host using ftp. |
| # 8. Transfer the dumpfile to another host using smb. |
| # |
| # See also 'svn-backup-dumps.py -h'. |
| # |
| # |
| # 1. Create a full dump (revisions 0 to HEAD). |
| # |
| # svn-backup-dumps.py <repos> <dumpdir> |
| # |
| # <repos> Path to the repository. |
| # <dumpdir> Directory for storing the dump file. |
| # |
| # This creates a dump file named 'src.000000-NNNNNN.svndmp.gz' |
| # where NNNNNN is the revision number of HEAD. |
| # |
| # |
| # 2. Create incremental dumps containing at most N revisions. |
| # |
| # svn-backup-dumps.py -c <count> <repos> <dumpdir> |
| # |
| # <count> Count of revisions per dump file. |
| # <repos> Path to the repository. |
| # <dumpdir> Directory for storing the dump file. |
| # |
| # When started the first time with a count of 1000 and if HEAD is |
| # at 2923 it creates the following files: |
| # |
| # src.000000-000999.svndmp.gz |
| # src.001000-001999.svndmp.gz |
| # src.002000-002923.svndmp.gz |
| # |
| # Say the next time HEAD is at 3045 it creates these two files: |
| # |
| # src.002000-002999.svndmp.gz |
| # src.003000-003045.svndmp.gz |
| # |
| # |
| # 3. Create incremental single revision dumps (for use in post-commit). |
| # |
| # svn-backup-dumps.py -r <revnr> <repos> <dumpdir> |
| # |
| # <revnr> A revision number. |
| # <repos> Path to the repository. |
| # <dumpdir> Directory for storing the dump file. |
| # |
| # This creates a dump file named 'src.NNNNNN.svndmp.gz' where |
| # NNNNNN is the revision number of HEAD. |
| # |
| # |
| # 4. Create incremental dumps relative to last dump |
| # |
| # svn-backup-dumps.py -i <repos> <dumpdir> |
| # |
| # <repos> Path to the repository. |
| # <dumpdir> Directory for storing the dump file. |
| # |
| # When if dumps are performed when HEAD is 2923, |
| # then when HEAD is 3045, is creates these files: |
| # |
| # src.000000-002923.svndmp.gz |
| # src.002924-003045.svndmp.gz |
| # |
| # |
| # 5. Create gzipped dump files. |
| # |
| # svn-backup-dumps.py -z ... |
| # |
| # ... More options, see 1-4, 7, 8. |
| # |
| # |
| # 6. Create bzipped dump files. |
| # |
| # svn-backup-dumps.py -b ... |
| # |
| # ... More options, see 1-4, 7, 8. |
| # |
| # |
| # 7. Transfer the dumpfile to another host using ftp. |
| # |
| # svn-backup-dumps.py -t ftp:<host>:<user>:<password>:<path> ... |
| # |
| # <host> Name of the FTP host. |
| # <user> Username on the remote host. |
| # <password> Password for the user. |
| # <path> Subdirectory on the remote host. |
| # ... More options, see 1-6. |
| # |
| # If <path> contains the string '%r' it is replaced by the |
| # repository name (basename of the repository path). |
| # |
| # |
| # 8. Transfer the dumpfile to another host using smb. |
| # |
| # svn-backup-dumps.py -t smb:<share>:<user>:<password>:<path> ... |
| # |
| # <share> Name of an SMB share in the form '//host/share'. |
| # <user> Username on the remote host. |
| # <password> Password for the user. |
| # <path> Subdirectory of the share. |
| # ... More options, see 1-6. |
| # |
| # If <path> contains the string '%r' it is replaced by the |
| # repository name (basename of the repository path). |
| # |
| # |
| # |
| # TODO: |
| # - find out how to report smbclient errors |
| # - improve documentation |
| # |
| |
| __version = "0.6" |
| |
| import sys |
| import os |
| if os.name != "nt": |
| import fcntl |
| import select |
| import gzip |
| import os.path |
| import re |
| from optparse import OptionParser |
| from ftplib import FTP |
| from subprocess import Popen, PIPE |
| |
| try: |
| import bz2 |
| have_bz2 = True |
| except ImportError: |
| have_bz2 = False |
| |
| |
| class SvnBackupOutput: |
| |
| def __init__(self, abspath, filename): |
| self.__filename = filename |
| self.__absfilename = os.path.join(abspath, filename) |
| |
| def open(self): |
| pass |
| |
| def write(self, data): |
| pass |
| |
| def close(self): |
| pass |
| |
| def get_filename(self): |
| return self.__filename |
| |
| def get_absfilename(self): |
| return self.__absfilename |
| |
| |
| class SvnBackupOutputPlain(SvnBackupOutput): |
| |
| def __init__(self, abspath, filename): |
| SvnBackupOutput.__init__(self, abspath, filename) |
| |
| def open(self): |
| self.__ofd = open(self.get_absfilename(), "wb") |
| |
| def write(self, data): |
| self.__ofd.write(data) |
| |
| def close(self): |
| self.__ofd.close() |
| |
| |
| class SvnBackupOutputGzip(SvnBackupOutput): |
| |
| def __init__(self, abspath, filename): |
| SvnBackupOutput.__init__(self, abspath, filename + ".gz") |
| |
| def open(self): |
| self.__compressor = gzip.GzipFile(filename=self.get_absfilename(), |
| mode="wb") |
| |
| def write(self, data): |
| self.__compressor.write(data) |
| |
| def close(self): |
| self.__compressor.flush() |
| self.__compressor.close() |
| |
| |
| class SvnBackupOutputBzip2(SvnBackupOutput): |
| |
| def __init__(self, abspath, filename): |
| SvnBackupOutput.__init__(self, abspath, filename + ".bz2") |
| |
| def open(self): |
| self.__compressor = bz2.BZ2Compressor() |
| self.__ofd = open(self.get_absfilename(), "wb") |
| |
| def write(self, data): |
| self.__ofd.write(self.__compressor.compress(data)) |
| |
| def close(self): |
| self.__ofd.write(self.__compressor.flush()) |
| self.__ofd.close() |
| |
| class SvnBackupOutputCommand(SvnBackupOutput): |
| |
| def __init__(self, abspath, filename, file_extension, cmd_path, |
| cmd_options): |
| SvnBackupOutput.__init__(self, abspath, filename + file_extension) |
| self.__cmd_path = cmd_path |
| self.__cmd_options = cmd_options |
| |
| def open(self): |
| cmd = [ self.__cmd_path, self.__cmd_options ] |
| |
| self.__ofd = open(self.get_absfilename(), "wb") |
| try: |
| proc = Popen(cmd, stdin=PIPE, stdout=self.__ofd, shell=False) |
| except: |
| print((256, "", "Popen failed (%s ...):\n %s" % (cmd[0], |
| str(sys.exc_info()[1])))) |
| sys.exit(256) |
| self.__proc = proc |
| self.__stdin = proc.stdin |
| |
| def write(self, data): |
| self.__stdin.write(data) |
| |
| def close(self): |
| self.__stdin.close() |
| rc = self.__proc.wait() |
| self.__ofd.close() |
| |
| class SvnBackupException(Exception): |
| |
| def __init__(self, errortext): |
| self.errortext = errortext |
| |
| def __str__(self): |
| return self.errortext |
| |
| class SvnBackup: |
| |
| def __init__(self, options, args): |
| # need 3 args: progname, reposname, dumpdir |
| if len(args) != 3: |
| if len(args) < 3: |
| raise SvnBackupException("too few arguments, specify" |
| " repospath and dumpdir.\nuse -h or" |
| " --help option to see help.") |
| else: |
| raise SvnBackupException("too many arguments, specify" |
| " repospath and dumpdir only.\nuse" |
| " -h or --help option to see help.") |
| self.__repospath = args[1] |
| self.__dumpdir = args[2] |
| # check repospath |
| rpathparts = os.path.split(self.__repospath) |
| if len(rpathparts[1]) == 0: |
| # repospath without trailing slash |
| self.__repospath = rpathparts[0] |
| if not os.path.exists(self.__repospath): |
| raise SvnBackupException("repos '%s' does not exist." % self.__repospath) |
| if not os.path.isdir(self.__repospath): |
| raise SvnBackupException("repos '%s' is not a directory." % self.__repospath) |
| for subdir in [ "db", "conf", "hooks" ]: |
| dir = os.path.join(self.__repospath, subdir) |
| if not os.path.isdir(dir): |
| raise SvnBackupException("repos '%s' is not a repository." % self.__repospath) |
| rpathparts = os.path.split(self.__repospath) |
| self.__reposname = rpathparts[1] |
| if self.__reposname in [ "", ".", ".." ]: |
| raise SvnBackupException("couldn't extract repos name from '%s'." % self.__repospath) |
| # check dumpdir |
| if not os.path.exists(self.__dumpdir): |
| raise SvnBackupException("dumpdir '%s' does not exist." % self.__dumpdir) |
| elif not os.path.isdir(self.__dumpdir): |
| raise SvnBackupException("dumpdir '%s' is not a directory." % self.__dumpdir) |
| # set options |
| self.__rev_nr = options.rev |
| self.__count = options.cnt |
| self.__quiet = options.quiet |
| self.__deltas = options.deltas |
| self.__relative_incremental = options.relative_incremental |
| |
| # svnadmin/svnlook path |
| self.__svnadmin_path = "svnadmin" |
| if options.svnadmin_path: |
| self.__svnadmin_path = options.svnadmin_path |
| self.__svnlook_path = "svnlook" |
| if options.svnlook_path: |
| self.__svnlook_path = options.svnlook_path |
| |
| # check compress option |
| self.__gzip_path = options.gzip_path |
| self.__bzip2_path = options.bzip2_path |
| self.__zip = None |
| compress_options = 0 |
| if options.gzip_path != None: |
| compress_options = compress_options + 1 |
| if options.bzip2_path != None: |
| compress_options = compress_options + 1 |
| if options.bzip2: |
| compress_options = compress_options + 1 |
| self.__zip = "bzip2" |
| if options.gzip: |
| compress_options = compress_options + 1 |
| self.__zip = "gzip" |
| if compress_options > 1: |
| raise SvnBackupException("--bzip2-path, --gzip-path, -b, -z are " |
| "mutually exclusive.") |
| |
| self.__overwrite = False |
| self.__overwrite_all = False |
| if options.overwrite > 0: |
| self.__overwrite = True |
| if options.overwrite > 1: |
| self.__overwrite_all = True |
| self.__transfer = None |
| if options.transfer != None: |
| self.__transfer = options.transfer.split(":") |
| if len(self.__transfer) != 5: |
| if len(self.__transfer) < 5: |
| raise SvnBackupException("too few fields for transfer '%s'." % self.__transfer) |
| else: |
| raise SvnBackupException("too many fields for transfer '%s'." % self.__transfer) |
| if self.__transfer[0] not in [ "ftp", "smb" ]: |
| raise SvnBackupException("unknown transfer method '%s'." % self.__transfer[0]) |
| |
| def set_nonblock(self, fileobj): |
| fd = fileobj.fileno() |
| n = fcntl.fcntl(fd, fcntl.F_GETFL) |
| fcntl.fcntl(fd, fcntl.F_SETFL, n|os.O_NONBLOCK) |
| |
| def exec_cmd(self, cmd, output=None, printerr=False): |
| if os.name == "nt": |
| return self.exec_cmd_nt(cmd, output, printerr) |
| else: |
| return self.exec_cmd_unix(cmd, output, printerr) |
| |
| def exec_cmd_unix(self, cmd, output=None, printerr=False): |
| try: |
| proc = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=False) |
| except: |
| return (256, "", "Popen failed (%s ...):\n %s" % (cmd[0], |
| str(sys.exc_info()[1]))) |
| stdout = proc.stdout |
| stderr = proc.stderr |
| self.set_nonblock(stdout) |
| self.set_nonblock(stderr) |
| readfds = [ stdout, stderr ] |
| selres = select.select(readfds, [], []) |
| bufout = "" |
| buferr = "" |
| while len(selres[0]) > 0: |
| for fd in selres[0]: |
| buf = fd.read(16384) |
| if len(buf) == 0: |
| readfds.remove(fd) |
| elif fd == stdout: |
| if output: |
| output.write(buf) |
| else: |
| bufout += buf |
| else: |
| if printerr: |
| sys.stdout.write("%s " % buf) |
| else: |
| buferr += buf |
| if len(readfds) == 0: |
| break |
| selres = select.select(readfds, [], []) |
| rc = proc.wait() |
| if printerr: |
| print("") |
| return (rc, bufout, buferr) |
| |
| def exec_cmd_nt(self, cmd, output=None, printerr=False): |
| try: |
| proc = Popen(cmd, stdout=PIPE, stderr=None, shell=False) |
| except: |
| return (256, "", "Popen failed (%s ...):\n %s" % (cmd[0], |
| str(sys.exc_info()[1]))) |
| stdout = proc.stdout |
| bufout = "" |
| buferr = "" |
| buf = stdout.read(16384) |
| while len(buf) > 0: |
| if output: |
| output.write(buf) |
| else: |
| bufout += buf |
| buf = stdout.read(16384) |
| rc = proc.wait() |
| return (rc, bufout, buferr) |
| |
| def get_head_rev(self): |
| cmd = [ self.__svnlook_path, "youngest", self.__repospath ] |
| r = self.exec_cmd(cmd) |
| if r[0] == 0 and len(r[2]) == 0: |
| return int(r[1].strip()) |
| else: |
| print(r[2]) |
| return -1 |
| |
| def get_last_dumped_rev(self): |
| filename_regex = re.compile("(.+)\.\d+-(\d+)\.svndmp.*") |
| # start with -1 so the next one will be rev 0 |
| highest_rev = -1 |
| |
| for filename in os.listdir(self.__dumpdir): |
| m = filename_regex.match( filename ) |
| if m and (m.group(1) == self.__reposname): |
| rev_end = int(m.group(2)) |
| |
| if rev_end > highest_rev: |
| # determine the latest revision dumped |
| highest_rev = rev_end |
| |
| return highest_rev |
| |
| def transfer_ftp(self, absfilename, filename): |
| rc = False |
| try: |
| host = self.__transfer[1] |
| user = self.__transfer[2] |
| passwd = self.__transfer[3] |
| destdir = self.__transfer[4].replace("%r", self.__reposname) |
| ftp = FTP(host, user, passwd) |
| ftp.cwd(destdir) |
| ifd = open(absfilename, "rb") |
| ftp.storbinary("STOR %s" % filename, ifd) |
| ftp.quit() |
| rc = len(ifd.read(1)) == 0 |
| ifd.close() |
| except Exception as e: |
| raise SvnBackupException("ftp transfer failed:\n file: '%s'\n error: %s" % \ |
| (absfilename, str(e))) |
| return rc |
| |
| def transfer_smb(self, absfilename, filename): |
| share = self.__transfer[1] |
| user = self.__transfer[2] |
| passwd = self.__transfer[3] |
| if passwd == "": |
| passwd = "-N" |
| destdir = self.__transfer[4].replace("%r", self.__reposname) |
| cmd = ("smbclient", share, "-U", user, passwd, "-D", destdir, |
| "-c", "put %s %s" % (absfilename, filename)) |
| r = self.exec_cmd(cmd) |
| rc = r[0] == 0 |
| if not rc: |
| print(r[2]) |
| return rc |
| |
| def transfer(self, absfilename, filename): |
| if self.__transfer == None: |
| return |
| elif self.__transfer[0] == "ftp": |
| self.transfer_ftp(absfilename, filename) |
| elif self.__transfer[0] == "smb": |
| self.transfer_smb(absfilename, filename) |
| else: |
| print("unknown transfer method '%s'." % self.__transfer[0]) |
| |
| def create_dump(self, checkonly, overwrite, fromrev, torev=None): |
| revparam = "%d" % fromrev |
| r = "%06d" % fromrev |
| if torev != None: |
| revparam += ":%d" % torev |
| r += "-%06d" % torev |
| filename = "%s.%s.svndmp" % (self.__reposname, r) |
| output = None |
| if self.__bzip2_path: |
| output = SvnBackupOutputCommand(self.__dumpdir, filename, ".bz2", |
| self.__bzip2_path, "-cz" ) |
| elif self.__gzip_path: |
| output = SvnBackupOutputCommand(self.__dumpdir, filename, ".gz", |
| self.__gzip_path, "-cf" ) |
| elif self.__zip: |
| if self.__zip == "gzip": |
| output = SvnBackupOutputGzip(self.__dumpdir, filename) |
| else: |
| output = SvnBackupOutputBzip2(self.__dumpdir, filename) |
| else: |
| output = SvnBackupOutputPlain(self.__dumpdir, filename) |
| absfilename = output.get_absfilename() |
| realfilename = output.get_filename() |
| if checkonly: |
| return os.path.exists(absfilename) |
| elif os.path.exists(absfilename): |
| if overwrite: |
| print("overwriting " + absfilename) |
| else: |
| print("%s already exists." % absfilename) |
| return True |
| else: |
| print("writing " + absfilename) |
| cmd = [ self.__svnadmin_path, "dump", |
| "--incremental", "-r", revparam, self.__repospath ] |
| if self.__quiet: |
| cmd[2:2] = [ "-q" ] |
| if self.__deltas: |
| cmd[2:2] = [ "--deltas" ] |
| output.open() |
| r = self.exec_cmd(cmd, output, True) |
| output.close() |
| rc = r[0] == 0 |
| if rc: |
| self.transfer(absfilename, realfilename) |
| return rc |
| |
| def export_single_rev(self): |
| return self.create_dump(False, self.__overwrite, self.__rev_nr) |
| |
| def export(self): |
| headrev = self.get_head_rev() |
| if headrev == -1: |
| return False |
| if self.__count is None: |
| return self.create_dump(False, self.__overwrite, 0, headrev) |
| baserev = headrev - (headrev % self.__count) |
| rc = True |
| cnt = self.__count |
| fromrev = baserev - cnt |
| torev = baserev - 1 |
| while fromrev >= 0 and rc: |
| if self.__overwrite_all or \ |
| not self.create_dump(True, False, fromrev, torev): |
| rc = self.create_dump(False, self.__overwrite_all, |
| fromrev, torev) |
| fromrev -= cnt |
| torev -= cnt |
| else: |
| fromrev = -1 |
| if rc: |
| rc = self.create_dump(False, self.__overwrite, baserev, headrev) |
| return rc |
| |
| def export_relative_incremental(self): |
| headrev = self.get_head_rev() |
| if headrev == -1: |
| return False |
| |
| last_dumped_rev = self.get_last_dumped_rev(); |
| if headrev < last_dumped_rev: |
| # that should not happen... |
| return False |
| |
| if headrev == last_dumped_rev: |
| # already up-to-date |
| return True |
| |
| return self.create_dump(False, False, last_dumped_rev + 1, headrev) |
| |
| def execute(self): |
| if self.__rev_nr != None: |
| return self.export_single_rev() |
| elif self.__relative_incremental: |
| return self.export_relative_incremental() |
| else: |
| return self.export() |
| |
| |
| if __name__ == "__main__": |
| usage = "usage: svn-backup-dumps.py [options] repospath dumpdir" |
| parser = OptionParser(usage=usage, version="%prog "+__version) |
| if have_bz2: |
| parser.add_option("-b", |
| action="store_true", |
| dest="bzip2", default=False, |
| help="compress the dump using python bzip2 library.") |
| parser.add_option("-i", |
| action="store_true", |
| dest="relative_incremental", default=False, |
| help="perform incremental relative to last dump.") |
| parser.add_option("--deltas", |
| action="store_true", |
| dest="deltas", default=False, |
| help="pass --deltas to svnadmin dump.") |
| parser.add_option("-c", |
| action="store", type="int", |
| dest="cnt", default=None, |
| help="count of revisions per dumpfile.") |
| parser.add_option("-o", |
| action="store_const", const=1, |
| dest="overwrite", default=0, |
| help="overwrite files.") |
| parser.add_option("-O", |
| action="store_const", const=2, |
| dest="overwrite", default=0, |
| help="overwrite all files.") |
| parser.add_option("-q", |
| action="store_true", |
| dest="quiet", default=False, |
| help="quiet.") |
| parser.add_option("-r", |
| action="store", type="int", |
| dest="rev", default=None, |
| help="revision number for single rev dump.") |
| parser.add_option("-t", |
| action="store", type="string", |
| dest="transfer", default=None, |
| help="transfer dumps to another machine "+ |
| "(s.a. --help-transfer).") |
| parser.add_option("-z", |
| action="store_true", |
| dest="gzip", default=False, |
| help="compress the dump using python gzip library.") |
| parser.add_option("--bzip2-path", |
| action="store", type="string", |
| dest="bzip2_path", default=None, |
| help="compress the dump using bzip2 custom command.") |
| parser.add_option("--gzip-path", |
| action="store", type="string", |
| dest="gzip_path", default=None, |
| help="compress the dump using gzip custom command.") |
| parser.add_option("--svnadmin-path", |
| action="store", type="string", |
| dest="svnadmin_path", default=None, |
| help="svnadmin command path.") |
| parser.add_option("--svnlook-path", |
| action="store", type="string", |
| dest="svnlook_path", default=None, |
| help="svnlook command path.") |
| parser.add_option("--help-transfer", |
| action="store_true", |
| dest="help_transfer", default=False, |
| help="shows detailed help for the transfer option.") |
| (options, args) = parser.parse_args(sys.argv) |
| if options.help_transfer: |
| print("Transfer help:") |
| print("") |
| print(" FTP:") |
| print(" -t ftp:<host>:<user>:<password>:<dest-path>") |
| print("") |
| print(" SMB (using smbclient):") |
| print(" -t smb:<share>:<user>:<password>:<dest-path>") |
| print("") |
| sys.exit(0) |
| rc = False |
| try: |
| backup = SvnBackup(options, args) |
| rc = backup.execute() |
| except SvnBackupException as e: |
| print("svn-backup-dumps.py: %s" % e) |
| if rc: |
| print("Everything OK.") |
| sys.exit(0) |
| else: |
| print("An error occurred!") |
| sys.exit(1) |
| |
| # vim:et:ts=4:sw=4 |
