| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """ Workaround for slow updates from an svn branch to git. |
| See also jira issue INFRA-9182 |
| |
| Situation: |
| |
| Remote svn repo ---> (slow) git-svn fetch ---> Remote git repo (upstream) |
| | | |
| | | |
| v v |
| Local svn working copy ---> this workaround ---> Local git repo |
| |
| When the remote git-svn fetch is slow, the remote git repo is behind |
| the remote svn repo. |
| |
| When this script is run it will first check that the local working copy and repository are clean. |
| Then it switches the svn working copy to the branch, which updates from the remote. |
| Then it the fetches branch from the git upstream repo, and merges the branch locally. |
| Normally the local svn and git will then be at the same svn revision, and the script will exit. |
| |
| Otherwise the remote git repo is out of date, and the following happens. |
| |
| For the branch branchname in a local git repository following an upstream git-svn git repository, |
| this maintains commits on a temporary git branch branchname.svn in the local git repository. |
| These commits contain a message ending like this: |
| "SvnRepoUrl diff -r EarlierSvnRevisionNumber:NextSvnRevisionNumber". |
| Otherwise the messages of the added commits are the same as their counterparts from git svn. |
| |
| Normally the added git commits and their git-svn counterparts have no differences between their working trees. |
| However such differences can occur, see also the documentation of git-svn reset and the limitations below. |
| In order not to interfere with git-svn this script only adds commits to a temporary branch |
| branchname.svn, and the commit messages are chosen differently, they do not contain git-svn-id. |
| |
| In case an earlier branchname.svn exists, it will first be deleted if necessary, |
| and restarted at the later branch. |
| Therefore branchname.svn is temporary and should only be used locally. |
| |
| By default, no more than 20 commits will be added to branchname.svn in a single run. |
| |
| The earlier revision number is taken from the git-svn-id message of git svn, |
| or from the latest revision number in the commit message on branchname.svn, |
| whichever is later. |
| |
| This allows branchname.svn to be used as a local git branch instead of branchname |
| to develop new features locally, for example by merging branchname.svn into a feature branch. |
| """ |
| |
| """ Limitations: |
| |
| This currently works by patching text, and therefore this does not work on binary files. |
| An example commit in lucene-solr that adds a binary file, on which this currently does not work correctly: |
| svn revision 1707457 |
| git commit 3c0390f71e1f08a17f32bc207b4003362f8b6ac2 |
| |
| When the local svn working copy contains file after updating to the latest available revision, |
| and there is an interim commit that deletes this file, this file is left as an empty file in the working directory |
| of the local git repository. |
| |
| All svn properties are ignored here. |
| """ |
| |
| """ To be done: |
| Take binary files from the patch, and check out binary files directly from the remote svn repo directly into the local git repo. |
| |
| Going really far: checkout each interim svn revision, and use all (changed) files from there instead of the text diff. |
| Determining all files under version control with svn (svn ls) is far too slow for this (esp. when compared to git ls-tree), |
| so this is probably better done by using svnsync to setup a local mirror repository following the remote, |
| and then using svnlook on the local mirror repository. |
| Doing that only to avoid the limitations of this workaround does not appear to be worthwhile. |
| """ |
| |
| """ This was developed on Linux using the following program versions: |
| python 2.7.6 |
| git 1.9.1 |
| svn 1.8.8 |
| grep (GNU grep) 2.16 |
| |
| gitk (part of git) was used for manual testing: |
| - reset branch to an earlier commit to simulate a non working update from svn to git, |
| - delete branchname.svn, reset branchname.svn to earlier, |
| - diff a commit generated here to a commit from git svn, |
| - update, reload, show commits by commit date, ... |
| """ |
| |
| import os |
| import subprocess |
| |
| from xml import sax |
| from xml.sax.handler import ContentHandler |
| |
| import types |
| |
| class SvnInfoHandler(ContentHandler): |
| revisionAttr = "revision" |
| |
| def __init__(self): |
| self.lastChangeRev = None |
| |
| def startElement(self, name, attrs): |
| if name == "commit": |
| self.lastChangeRev = int(attrs.getValue(self.revisionAttr)) |
| |
| def getLastChangeRevision(self): |
| return self.lastChangeRev |
| |
| |
| class SvnLogEntry(object): |
| pass # attributes set in SvnLogHandler: revision, author, date, msg |
| |
| |
| class SvnLogHandler(ContentHandler): # collect list of SvnLogEntry's |
| logEntryTag = "logentry" |
| revisionAttr = "revision" # also used as SvnLogEntry attribute |
| authorTag = "author" |
| dateTag = "date" |
| msgTag = "msg" |
| charCollectTags = (authorTag, dateTag, msgTag) # also used as SvnLogEntry attributes |
| |
| def __init__(self): |
| self.logEntries = [] |
| self.chars = None |
| |
| def startElement(self, name, attrs): |
| if name == self.logEntryTag: |
| self.lastLogEntry = SvnLogEntry() |
| setattr(self.lastLogEntry, self.revisionAttr, int(attrs.getValue(self.revisionAttr))) |
| for tag in self.charCollectTags: |
| setattr(self.lastLogEntry, tag, None) |
| return |
| |
| if name in self.charCollectTags: |
| self.chars = "" |
| |
| def characters(self, content): |
| if self.chars is not None: |
| self.chars += content |
| |
| def endElement(self, name): |
| if name in self.charCollectTags: |
| setattr(self.lastLogEntry, name, self.chars) |
| self.chars = None |
| return |
| |
| if name == self.logEntryTag: |
| self.logEntries.append(self.lastLogEntry) |
| self.lastLogEntry = None |
| |
| def getLogEntries(self): |
| return self.logEntries |
| |
| |
| class SubProcessAtPath(object): |
| def __init__(self, pathName, verbose=True): |
| assert pathName != "" |
| self.pathName = pathName |
| self.verbose = verbose |
| |
| def getPathName(self): |
| return self.pathName |
| |
| def chDirToPath(self): |
| if self.pathName != os.getcwd(): |
| os.chdir(self.pathName) |
| assert self.pathName == os.getcwd() |
| |
| def __str__(self): |
| return self.__class__.__name__ + "(" + self.pathName + ")" |
| |
| def checkCall(self, *args, **kwArgs): |
| assert type(*args) != types.StringType |
| self.chDirToPath() |
| if self.verbose: |
| print "check_call args:", " ".join(*args) |
| subprocess.check_call(*args, **kwArgs) |
| |
| def checkOutput(self, *args, **kwArgs): |
| assert type(*args) != types.StringType |
| self.chDirToPath() |
| if self.verbose: |
| print "check_output args:", " ".join(*args) |
| result = subprocess.check_output(*args, **kwArgs) |
| if self.verbose: |
| print "check_output result:", result |
| return result |
| |
| |
| class SvnWorkingCopy(SubProcessAtPath): |
| def __init__(self, pathName): |
| SubProcessAtPath.__init__(self, pathName, verbose=False) |
| |
| svnCmd = "svn" |
| |
| def ensureNoLocalModifications(self): |
| localMods = self.checkOutput((self.svnCmd, "status")) |
| if localMods: |
| errorExit(self, "should not have local modifications:\n", localMods) |
| |
| def update(self): |
| self.checkCall((self.svnCmd, "update")) |
| |
| def switch(self, repoBranchName): |
| self.checkCall((self.svnCmd, "switch", ("^/" + repoBranchName))) |
| |
| def lastChangedRevision(self): |
| infoXml = self.checkOutput((self.svnCmd, "info", "--xml")) |
| infoHandler = SvnInfoHandler() |
| sax.parseString(infoXml, infoHandler) |
| return infoHandler.getLastChangeRevision() |
| |
| def getLogEntries(self, fromRevision, toRevision, maxNumLogEntries): |
| revRange = self.revisionsRange(fromRevision, toRevision) |
| logXml = self.checkOutput((self.svnCmd, "log", "-r", revRange, "--xml", "-l", str(maxNumLogEntries))) |
| logHandler = SvnLogHandler() |
| sax.parseString(logXml, logHandler) |
| return logHandler.getLogEntries() |
| |
| def revisionsRange(self, fromRevision, toRevision): |
| return str(fromRevision) + ":" + str(toRevision) |
| |
| def createPatchFile(self, fromRevision, toRevision, patchFileName): |
| revRange = self.revisionsRange(fromRevision, toRevision) |
| patchFile = open(patchFileName, 'w') |
| try: |
| print "Creating patch from", self.pathName, "between revisions", revRange |
| self.checkCall((self.svnCmd, "diff", "-r", revRange, |
| "--ignore-properties"), # git apply can fail on svn properties. |
| stdout=patchFile) |
| finally: |
| patchFile.close() |
| print "Created patch file", patchFileName |
| |
| def patchedFileNames(self, patchFileName): # return a sequence of the patched file names |
| if os.path.getsize(patchFileName) == 0: # changed only svn properties, no files changed. |
| return [] |
| |
| indexPrefix = "Index: " |
| regExp = "^" + indexPrefix # at beginning of line |
| patchedFileNamesLines = self.checkOutput(("grep", regExp, patchFileName)) # grep exits 1 whithout any match. |
| indexPrefixLength = len(indexPrefix) |
| return [line[indexPrefixLength:] |
| for line in patchedFileNamesLines.split("\n") |
| if len(line) > 0] |
| |
| |
| class GitRepository(SubProcessAtPath): |
| def __init__(self, pathName): |
| SubProcessAtPath.__init__(self, pathName, verbose=False) |
| self.currentBranch = None |
| |
| gitCmd = "git" |
| |
| def checkOutBranch(self, branchName): |
| self.checkCall((self.gitCmd, "checkout", branchName)) |
| self.currentBranch = branchName |
| |
| def getCurrentBranch(self): |
| if self.currentBranch is None: |
| gitStatusOut = self.checkOutput((self.gitCmd, "status")) |
| if gitStatusOut.startswith("On branch "): |
| self.currentBranch = gitStatusOut.split[2] |
| else: |
| errorExit(self, "not on a branch:", gitStatusOut) |
| return self.currentBranch |
| |
| def workingDirectoryClean(self): |
| gitStatusOut = self.checkOutput((self.gitCmd, "status")) |
| expSubString = "nothing to commit, working directory clean" |
| return gitStatusOut.find(expSubString) >= 0 |
| |
| def listBranches(self, pattern): |
| return self.checkOutput((self.gitCmd, "branch", "--list", pattern)) |
| |
| def branchExists(self, branchName): |
| listOut = self.listBranches(branchName) # CHECKME: using branchName as pattern may not always be ok. |
| return len(listOut) > 0 |
| |
| def deleteBranch(self, branchName): |
| self.checkCall((self.gitCmd, "branch", "-D", branchName)) |
| if branchName == self.currentBranch: |
| self.currentBranch = None |
| |
| def createBranch(self, branchName): |
| self.checkCall((self.gitCmd, "branch", branchName)) |
| |
| def fetch(self, upStream): |
| self.checkCall((self.gitCmd, "fetch", upStream)) |
| |
| def merge(self, branch, fromBranch): |
| self.checkCall((self.gitCmd, "merge", branch, fromBranch)) |
| |
| def getCommitMessage(self, commitRef): |
| return self.checkOutput((self.gitCmd, "log", "--format=%B", "-n", "1", commitRef)) |
| |
| def getCommitAuthorName(self, commitRef): |
| return self.checkOutput((self.gitCmd, "log", "--format=%aN", "-n", "1", commitRef)) |
| |
| def getCommitAuthorEmail(self, commitRef): |
| return self.checkOutput((self.gitCmd, "log", "--format=%aE", "-n", "1", commitRef)) |
| |
| def getLatestCommitForAuthor(self, svnAuthor): |
| authorCommit = self.checkOutput( |
| " ".join((self.gitCmd, |
| "rev-list", "--all", "-i", ("--author=" + svnAuthor), # see git commit documentation on --author |
| "|", # pipe should have a buffer for at most a few commit ids. |
| "head", "-1")), |
| shell=True) # use shell pipe |
| authorCommit = authorCommit.rstrip("\n") |
| return authorCommit |
| |
| def getSvnRemoteAndRevision(self, gitSvnCommitRef): |
| gitSvnCommitMessage = self.getCommitMessage(gitSvnCommitRef) |
| words = gitSvnCommitMessage.split(); |
| svnIdMarker = "git-svn-id:" |
| assert words.index(svnIdMarker) >= 0 |
| svnId = words[words.index(svnIdMarker) + 1] |
| splitSvnId = svnId.split("@") |
| svnRemote = splitSvnId[0] |
| svnRevision = int(splitSvnId[1]) |
| return (svnRemote, svnRevision) |
| |
| def lastTempGitSvnRevision(self, branchName): # at a commit generated here on the temp branch. |
| gitCommitMessage = self.getCommitMessage(branchName) |
| parts = gitCommitMessage.split(":") |
| lastPart = parts[-1].split()[0] # remove appended newlines |
| try: |
| return int(lastPart) |
| except: # not generated here, ignore. |
| print "Warning: svn revision range not found at end of commit message:\n", gitCommitMessage |
| return None |
| |
| def applyPatch(self, patchFileName, stripDepth): |
| self.checkCall((self.gitCmd, "apply", |
| ("-p" + str(stripDepth)), |
| "--whitespace=nowarn", |
| patchFileName)) |
| |
| def addAllToIndex(self): |
| self.checkCall((self.gitCmd, "add", "-A")) |
| |
| def deleteForced(self, fileName): |
| self.checkCall((self.gitCmd, "rm", "-f", fileName)) |
| |
| def commit(self, message, |
| authorName, authorEmail, authorDate, |
| committerName, committerEmail, committerDate): |
| author = ''.join((authorName, " <", authorEmail, ">")) |
| os.environ["GIT_COMMITTER_NAME"] = committerName # no need to save/restore earlier environment state. |
| os.environ["GIT_COMMITTER_EMAIL"] = committerEmail |
| os.environ["GIT_COMMITTER_DATE"] = committerDate |
| self.checkCall((self.gitCmd, "commit", |
| "--allow-empty", # only svn poperties changed. |
| ("--message=" + message), |
| ("--author=" + author), |
| ("--date=" + authorDate) )) |
| |
| def cleanDirsForced(self): |
| self.checkCall((self.gitCmd, "clean", "-fd")) |
| |
| |
| |
| def errorExit(*messageParts): |
| raise RuntimeError(" ".join(map(str, messageParts))) |
| |
| |
| def allSuccessivePairs(lst): |
| return [lst[i:i+2] for i in range(len(lst)-1)] |
| |
| |
| def maintainTempGitSvnBranch(branchName, tempGitBranchName, |
| svnWorkingCopyOfBranchPath, svnRepoBranchName, |
| gitRepoPath, gitUpstream, |
| patchFileName, |
| maxCommits=20, # generate at most this number of commits on tempGitBranchName, rerun to add more. |
| testMode=False): |
| |
| assert maxCommits >= 1 |
| |
| gitRepo = GitRepository(gitRepoPath) |
| gitRepo.checkOutBranch(branchName) # fails with git message when working directory is not clean |
| |
| svnWorkingCopy = SvnWorkingCopy(svnWorkingCopyOfBranchPath) |
| svnWorkingCopy.ensureNoLocalModifications() |
| svnWorkingCopy.switch(svnRepoBranchName) # switch to repo branch, update to latest revision |
| |
| lastSvnRevision = svnWorkingCopy.lastChangedRevision() |
| # print svnWorkingCopy, "lastSvnRevision:", lastSvnRevision |
| |
| gitRepo.fetch(gitUpstream) |
| if testMode: |
| pass # leave branch where it is, as if the last commits from upstream did not arrive |
| else: |
| gitRepo.merge(branchName, gitUpstream + "/" + branchName) |
| |
| (svnRemote, lastSvnRevisionOnGitSvnBranch) = gitRepo.getSvnRemoteAndRevision(branchName) |
| print "svnRemote:", svnRemote |
| #print gitRepo, branchName, "lastSvnRevisionOnGitSvnBranch:", lastSvnRevisionOnGitSvnBranch |
| |
| # check whether tempGitBranchName exists: |
| diffBaseRevision = lastSvnRevisionOnGitSvnBranch |
| svnTempRevision = None |
| doCommitOnExistingTempBranch = False |
| |
| if gitRepo.branchExists(tempGitBranchName): |
| print tempGitBranchName, "exists" |
| # update lastSvnRevisionOnGitSvnBranch from there. |
| svnTempRevision = gitRepo.lastTempGitSvnRevision(tempGitBranchName) |
| if svnTempRevision is None: |
| print "Warning: no svn revision found on branch:", tempGitBranchName |
| else: |
| if svnTempRevision > lastSvnRevisionOnGitSvnBranch: |
| diffBaseRevision = svnTempRevision |
| doCommitOnExistingTempBranch = True |
| gitRepo.checkOutBranch(tempGitBranchName) |
| |
| if lastSvnRevision == diffBaseRevision: |
| print gitRepo, gitRepo.getCurrentBranch(), "up to date with", svnWorkingCopy, svnRepoBranchName |
| return |
| |
| if lastSvnRevision < diffBaseRevision: # unlikely, do nothing |
| print gitRepo, gitRepo.getCurrentBranch(), "later than", svnWorkingCopy, ", nothing to update." |
| # CHECK: generate svn commits from the git commits? |
| return |
| |
| print gitRepo, gitRepo.getCurrentBranch(), "earlier than", svnWorkingCopy |
| |
| if not gitRepo.workingDirectoryClean(): |
| errorExit(gitRepo, "on branch", gitRepo.getCurrentBranch(), "not clean") |
| |
| print gitRepo,"on branch", gitRepo.getCurrentBranch(), "and clean" |
| |
| if not doCommitOnExistingTempBranch: # restart temp branch from branch |
| assert gitRepo.getCurrentBranch() == branchName |
| if gitRepo.branchExists(tempGitBranchName): # tempGitBranchName exists, delete it first. |
| print "Branch", tempGitBranchName, "exists, deleting" |
| gitRepo.deleteBranch(tempGitBranchName) |
| if gitRepo.branchExists(tempGitBranchName): |
| errorExit("Could not delete branch", tempGitBranchName, "from", gitRepo) |
| |
| gitRepo.createBranch(tempGitBranchName) |
| gitRepo.checkOutBranch(tempGitBranchName) |
| print "Started branch", tempGitBranchName, "at", branchName |
| |
| assert gitRepo.getCurrentBranch() == tempGitBranchName |
| |
| patchStripDepth = 0 # patch generated at svn repo. |
| |
| maxNumLogEntries = maxCommits + 1 |
| svnLogEntries = svnWorkingCopy.getLogEntries(diffBaseRevision, lastSvnRevision, maxNumLogEntries) |
| |
| numCommits = 0 |
| |
| for (logEntryFrom, logEntryTo) in allSuccessivePairs(svnLogEntries): |
| # create patch file from svn between the revisions: |
| svnWorkingCopy.createPatchFile(logEntryFrom.revision, logEntryTo.revision, patchFileName) |
| |
| patchedFileNames = svnWorkingCopy.patchedFileNames(patchFileName) |
| |
| if os.path.getsize(patchFileName) > 0: |
| gitRepo.applyPatch(patchFileName, patchStripDepth) |
| print "Applied patch", patchFileName |
| else: # only svn properties changed, do git commit for commit info only. |
| print "Empty patch", patchFileName |
| |
| gitRepo.addAllToIndex() # add all patch changes to the git index to be committed. |
| |
| # Applying the patch leaves files that have been actually deleted at zero size. |
| # Therefore delete empty patched files from the git repo that do not exist in svn working copy: |
| for patchedFileName in patchedFileNames: |
| fileNameInGitRepo = os.path.join(gitRepo.getPathName(), patchedFileName) |
| fileNameInSvnWorkingCopy = os.path.join(svnWorkingCopy.getPathName(), patchedFileName) |
| |
| if os.path.isdir(fileNameInGitRepo): |
| # print "Directory:", fileNameInGitRepo |
| continue |
| |
| if not os.path.isfile(fileNameInGitRepo): |
| print "Possibly new binary file in svn, ignored here:", fileNameInGitRepo |
| # FIXME: Take a new binary file out of the svn repository directly. |
| continue |
| |
| fileSize = os.path.getsize(fileNameInGitRepo) |
| if fileSize > 0: |
| # print "Non empty file patched normally:", fileNameInGitRepo |
| continue |
| |
| # fileNameInGitRepo exists and is empty |
| if os.path.isfile(fileNameInSvnWorkingCopy): |
| # FIXME: this only works correctly when the svn working copy is hecked out at the target revision. |
| print "Left empty file:", fileNameInGitRepo |
| continue |
| |
| gitRepo.deleteForced(fileNameInGitRepo) # force, the file is not up to date. This also stages the delete for commit. |
| # print "Deleted empty file", fileNameInGitRepo # not needed, git rm is verbose enough |
| |
| # commit, put toRevision at end so it can be picked up later. |
| revisionsRange = svnWorkingCopy.revisionsRange(logEntryFrom.revision, logEntryTo.revision) |
| message = logEntryTo.msg + "\n\n" + svnRemote + " diff -r " + revisionsRange |
| authorCommit = gitRepo.getLatestCommitForAuthor(logEntryTo.author) |
| authorName = gitRepo.getCommitAuthorName(authorCommit) |
| authorEmail = gitRepo.getCommitAuthorEmail(authorCommit) |
| # print "Author name and email:", authorName, authorEmail |
| gitRepo.commit(message, |
| authorName, authorEmail, logEntryTo.date, |
| authorName, authorEmail, logEntryTo.date) # author is also git committer, just like git-svn |
| |
| numCommits += 1 |
| |
| # print "Commit author:", logEntryTo.author |
| # print "Commit date:", logEntryTo.date |
| print "Commit message:", logEntryTo.msg |
| |
| gitRepo.cleanDirsForced() # delete untracked directories and files |
| |
| if not gitRepo.workingDirectoryClean(): |
| errorExit(gitRepo, "on branch", gitRepo.getCurrentBranch(), "not clean, numCommits:", numCommits) |
| |
| print "Added", numCommits, "commit(s) to branch", tempGitBranchName |
| |
| |
| if __name__ == "__main__": |
| |
| testMode = False # when true, leave branch where it is, as if the last commits from upstream did not arrive |
| defaultMaxCommits = 20 |
| maxCommits = defaultMaxCommits |
| |
| import sys |
| argv = sys.argv[1:] |
| while argv: |
| if argv[0] == "test": |
| testMode = True |
| else: |
| try: |
| maxCommits = int(argv[0]) |
| assert maxCommits >= 1 |
| except: |
| errorExit("Argument(s) should be test and/or a maximum number of commits, defaults are false and " + defaultMaxCommits) |
| argv = argv[1:] |
| |
| repo = "lucene-solr" |
| branchName = "trunk" |
| tempGitBranchName = branchName + ".svn" |
| |
| home = os.path.expanduser("~") |
| |
| svnWorkingCopyOfBranchPath = os.path.join(home, "svnwork", repo, branchName) |
| svnRepoBranchName = "lucene/dev/" + branchName # for svn switch to |
| |
| gitRepo = os.path.join(home, "gitrepos", repo) |
| gitUpstream = "upstream" |
| |
| patchFileName = os.path.join(home, "patches", tempGitBranchName) |
| |
| maintainTempGitSvnBranch(branchName, tempGitBranchName, |
| svnWorkingCopyOfBranchPath, svnRepoBranchName, |
| gitRepo, gitUpstream, |
| patchFileName, |
| maxCommits=maxCommits, |
| testMode=testMode) |
