Merge branch 'hs/7873'
diff --git a/Allura/allura/model/repo_refresh.py b/Allura/allura/model/repo_refresh.py
index fca220f..1ca82e8 100644
--- a/Allura/allura/model/repo_refresh.py
+++ b/Allura/allura/model/repo_refresh.py
@@ -124,6 +124,19 @@
if (i + 1) % 100 == 0:
log.info('Compute last commit info %d: %s', (i + 1), ci._id)
+ # Clear any existing caches for branches/tags
+ if repo.cached_branches:
+ repo.cached_branches = []
+ session(repo).flush()
+
+ if repo.cached_tags:
+ repo.cached_tags = []
+ session(repo).flush()
+ # The first view can be expensive to cache,
+ # so we want to do it here instead of on the first view.
+ repo.get_branches()
+ repo.get_tags()
+
if not all_commits and not new_clone:
for commit in commit_ids:
new = repo.commit(commit)
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index 022343f..a3d574c 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -349,6 +349,8 @@
repo_tags = FieldProperty(S.Deprecated)
upstream_repo = FieldProperty(dict(name=str, url=str))
default_branch_name = FieldProperty(str)
+ cached_branches = FieldProperty([dict(name=str, object_id=str)])
+ cached_tags = FieldProperty([dict(name=str, object_id=str)])
def __init__(self, **kw):
if 'name' in kw and 'tool' in kw:
diff --git a/Allura/development.ini b/Allura/development.ini
index 9146033..bb2ccab 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -297,6 +297,12 @@
scm.import.retry_count = 50
scm.import.retry_sleep_secs = 5
+; When getting a list of valid references (branches/tags) from a repo, you can cache
+; the results in mongo based on a threshold. Set `repo_refs_cache_threshold` (in seconds) and the resulting
+; lists will be cached and served from cache on subsequent requests until reset by `repo_refresh`.
+; Set to 0 to cache all references. Remove entirely to cache nothing.
+repo_refs_cache_threshold = .5
+
; One-click merge is enabled by default, but can be turned off on for each type of repo
scm.merge.git.disabled = false
scm.merge.hg.disabled = false
diff --git a/ForgeGit/forgegit/model/git_repo.py b/ForgeGit/forgegit/model/git_repo.py
index 416db33..7d87a09 100644
--- a/ForgeGit/forgegit/model/git_repo.py
+++ b/ForgeGit/forgegit/model/git_repo.py
@@ -532,6 +532,45 @@
except KeyError:
return False
+ def _get_refs(self, field_name):
+ """ Returns a list of valid reference objects (branches or tags) from the git database
+
+ :return: List of git ref objects.
+ :rtype: list
+ """
+
+ cache_name = 'cached_' + field_name
+ cache = getattr(self._repo, cache_name, None)
+
+ if cache:
+ return cache
+
+ refs = []
+ start_time = time()
+ ref_list = getattr(self._git, field_name)
+ for ref in ref_list:
+ try:
+ hex_sha = ref.commit.hexsha
+ except ValueError:
+ log.debug(u"Found invalid sha: {}".format(ref))
+ continue
+ refs.append(Object(name=ref.name, object_id=hex_sha))
+ time_taken = time() - start_time
+
+ threshold = tg.config.get('repo_refs_cache_threshold')
+ try:
+ threshold = float(threshold) if threshold else None
+ except ValueError:
+ threshold = None
+ log.warn('Skipping reference caching - The value for config param '
+ '"repo_refs_cache_threshold" must be a float.')
+
+ if threshold is not None and time_taken > threshold:
+ setattr(self._repo, cache_name, refs)
+ session(self._repo).flush(self._repo)
+
+ return refs
+
@LazyProperty
def head(self):
if not self._git or not self._git.heads:
@@ -549,15 +588,15 @@
@LazyProperty
def heads(self):
- return [Object(name=b.name, object_id=b.commit.hexsha) for b in self._git.heads if b.is_valid()]
+ return self._get_refs('heads')
@LazyProperty
def branches(self):
- return [Object(name=b.name, object_id=b.commit.hexsha) for b in self._git.branches if b.is_valid()]
+ return self._get_refs('branches')
@LazyProperty
def tags(self):
- return [Object(name=t.name, object_id=t.commit.hexsha) for t in self._git.tags if t.is_valid()]
+ return self._get_refs('tags')
def set_default_branch(self, name):
if not name:
diff --git a/ForgeGit/forgegit/tests/model/test_repository.py b/ForgeGit/forgegit/tests/model/test_repository.py
index 349efe6..c28daf1 100644
--- a/ForgeGit/forgegit/tests/model/test_repository.py
+++ b/ForgeGit/forgegit/tests/model/test_repository.py
@@ -728,6 +728,17 @@
res_with_tmp = self.repo.merge_request_commits(mr)
assert_equals(res_without_tmp, res_with_tmp)
+ def test_cached_branches(self):
+ with mock.patch.dict('allura.lib.app_globals.config', {'repo_refs_cache_threshold': '0'}):
+ rev = GM.Repository.query.get(_id=self.repo['_id'])
+ branches = rev._impl._get_refs('branches')
+ assert_equal(rev.cached_branches, branches)
+
+ def test_cached_tags(self):
+ with mock.patch.dict('allura.lib.app_globals.config', {'repo_refs_cache_threshold': '0'}):
+ rev = GM.Repository.query.get(_id=self.repo['_id'])
+ tags = rev._impl._get_refs('tags')
+ assert_equal(rev.cached_tags, tags)
class TestGitImplementation(unittest.TestCase):
@@ -735,6 +746,7 @@
repo_dir = pkg_resources.resource_filename(
'forgegit', 'tests/data/testgit.git')
repo = mock.Mock(full_fs_path=repo_dir)
+ repo.cached_branches = []
impl = GM.git_repo.GitImplementation(repo)
self.assertEqual(impl.branches, [
Object(name='master',
@@ -747,6 +759,7 @@
repo_dir = pkg_resources.resource_filename(
'forgegit', 'tests/data/testgit.git')
repo = mock.Mock(full_fs_path=repo_dir)
+ repo.cached_tags = []
impl = GM.git_repo.GitImplementation(repo)
self.assertEqual(impl.tags, [
Object(name='foo',
