[#4202] Clean up copy-pasted chunked_project_iterator usage.
Signed-off-by: Tim Van Steenburgh <tvansteenburgh@geek.net>
diff --git a/scripts/migrations/010-fix-home-permissions.py b/scripts/migrations/010-fix-home-permissions.py
index 05212da..f66e76a 100644
--- a/scripts/migrations/010-fix-home-permissions.py
+++ b/scripts/migrations/010-fix-home-permissions.py
@@ -7,6 +7,7 @@
from bson import ObjectId
from allura import model as M
+from allura.lib import utils
from forgewiki.wiki_main import ForgeWikiApp
log = logging.getLogger('fix-home-permissions')
@@ -22,8 +23,9 @@
else:
log.info('Fixing permissions for all Home Wikis')
- for some_projects in chunked_project_iterator({'neighborhood_id': {'$nin': [ObjectId('4be2faf8898e33156f00003e'), # /u
- ObjectId('4dbf2563bfc09e6362000005')]}}): # /motorola
+ for some_projects in utils.chunked_find(M.Project, {'neighborhood_id': {
+ '$nin': [ObjectId('4be2faf8898e33156f00003e'), # /u
+ ObjectId('4dbf2563bfc09e6362000005')]}}): # /motorola
for project in some_projects:
c.project = project
home_app = project.app_instance('home')
@@ -56,21 +58,6 @@
home_app.config.acl = map(dict, new_acl.values())
session(home_app.config).flush()
-PAGESIZE=1024
-
-def chunked_project_iterator(q_project):
- '''shamelessly copied from refresh-all-repos.py'''
- page = 0
- while True:
- results = (M.Project.query
- .find(q_project)
- .skip(PAGESIZE*page)
- .limit(PAGESIZE)
- .all())
- if not results: break
- yield results
- page += 1
-
def project_role(project, name):
role = M.ProjectRole.query.get(project_id=project._id, name=name)
if role is None:
diff --git a/scripts/migrations/011-fix-subroles.py b/scripts/migrations/011-fix-subroles.py
index c982a87..2d1e007 100644
--- a/scripts/migrations/011-fix-subroles.py
+++ b/scripts/migrations/011-fix-subroles.py
@@ -12,11 +12,11 @@
import sys
import logging
-from pylons import c
from ming.orm import session
from ming.orm.ormsession import ThreadLocalORMSession
from allura import model as M
+from allura.lib import utils
log = logging.getLogger('fix-subroles')
log.addHandler(logging.StreamHandler(sys.stdout))
@@ -27,7 +27,7 @@
log.info('Examining subroles in all non-user projects.')
n_users = M.Neighborhood.query.get(name='Users')
project_filter = dict(neighborhood_id={'$ne':n_users._id})
- for some_projects in chunked_project_iterator(project_filter):
+ for some_projects in utils.chunked_find(M.Project, project_filter):
for project in some_projects:
project_name = '%s.%s' % (project.neighborhood.name, project.shortname)
project_roles = {}
@@ -53,7 +53,7 @@
for user in project.users():
pr = user.project_role(project=project)
if not pr.roles: continue
- for parent, children in [('Admin', ('Developer', 'Member')),
+ for parent, children in [('Admin', ('Developer', 'Member')),
('Developer', ('Member',))]:
if project_roles[parent]._id not in pr.roles: continue
for role_name in children:
@@ -73,21 +73,5 @@
log.info('%s projects examined.' % num_projects_examined)
-
-PAGESIZE=1024
-
-def chunked_project_iterator(q_project):
- '''shamelessly copied from refresh-all-repos.py'''
- page = 0
- while True:
- results = (M.Project.query
- .find(q_project)
- .skip(PAGESIZE*page)
- .limit(PAGESIZE)
- .all())
- if not results: break
- yield results
- page += 1
-
if __name__ == '__main__':
main()
diff --git a/scripts/migrations/012-uninstall-home.py b/scripts/migrations/012-uninstall-home.py
index 7f1f9f5..a8b66f5 100644
--- a/scripts/migrations/012-uninstall-home.py
+++ b/scripts/migrations/012-uninstall-home.py
@@ -7,6 +7,7 @@
from mock import Mock, patch
from allura.lib import helpers as h
+from allura.lib import utils
from allura import model as M
from forgewiki import model as WM
from allura.ext.project_home import ProjectHomeApp
@@ -21,7 +22,8 @@
possibly_orphaned_projects = 0
solr_delete = Mock()
notification_post = Mock()
- for some_projects in chunked_project_iterator({'neighborhood_id': {'$ne': ObjectId("4be2faf8898e33156f00003e")}}):
+ for some_projects in utils.chunked_find(M.Project, {'neighborhood_id': {
+ '$ne': ObjectId("4be2faf8898e33156f00003e")}}):
for project in some_projects:
c.project = project
old_home_app = project.app_instance('home')
@@ -102,20 +104,5 @@
assert solr_delete.call_count == affected_projects, solr_delete.call_count
assert notification_post.call_count == 2 * affected_projects, notification_post.call_count
-PAGESIZE=1024
-
-def chunked_project_iterator(q_project):
- '''shamelessly copied from refresh-all-repos.py'''
- page = 0
- while True:
- results = (M.Project.query
- .find(q_project)
- .skip(PAGESIZE*page)
- .limit(PAGESIZE)
- .all())
- if not results: break
- yield results
- page += 1
-
if __name__ == '__main__':
main()
diff --git a/scripts/migrations/013-update-ordinals.py b/scripts/migrations/013-update-ordinals.py
index 9d8b13d..ef81d32 100644
--- a/scripts/migrations/013-update-ordinals.py
+++ b/scripts/migrations/013-update-ordinals.py
@@ -6,6 +6,7 @@
from ming.orm.ormsession import ThreadLocalORMSession
from allura import model as M
+from allura.lib import utils
log = logging.getLogger('update-ordinals')
log.addHandler(logging.StreamHandler(sys.stdout))
@@ -14,7 +15,7 @@
test = sys.argv[-1] == 'test'
num_projects_examined = 0
log.info('Examining all projects for mount order.')
- for some_projects in chunked_project_iterator({}):
+ for some_projects in utils.chunked_find(M.Project):
for project in some_projects:
c.project = project
mounts = project.ordered_mounts(include_search=True)
@@ -47,21 +48,5 @@
ThreadLocalORMSession.flush_all()
ThreadLocalORMSession.close_all()
-
-PAGESIZE=1024
-
-def chunked_project_iterator(q_project):
- '''shamelessly copied from refresh-all-repos.py'''
- page = 0
- while True:
- results = (M.Project.query
- .find(q_project)
- .skip(PAGESIZE*page)
- .limit(PAGESIZE)
- .all())
- if not results: break
- yield results
- page += 1
-
if __name__ == '__main__':
main()
diff --git a/scripts/refresh-all-repos.py b/scripts/refresh-all-repos.py
index c62d0f3..99cb4d1 100644
--- a/scripts/refresh-all-repos.py
+++ b/scripts/refresh-all-repos.py
@@ -1,11 +1,11 @@
import logging
import optparse
-from collections import defaultdict
from pylons import c
from ming.orm import ThreadLocalORMSession
from allura import model as M
+from allura.lib import utils
log = logging.getLogger(__name__)
@@ -45,7 +45,7 @@
M.repo.DiffInfoDoc.m.remove({})
M.repo.LastCommitDoc.m.remove({})
M.repo.CommitRunDoc.m.remove({})
- for chunk in chunked_project_iterator(q_project):
+ for chunk in utils.chunked_find(M.Project, q_project):
for p in chunk:
c.project = p
if projects:
@@ -73,18 +73,5 @@
ThreadLocalORMSession.flush_all()
ThreadLocalORMSession.close_all()
-def chunked_project_iterator(q_project):
- page = 0
- while True:
- results = (M.Project.query
- .find(q_project)
- .skip(PAGESIZE*page)
- .limit(PAGESIZE)
- .all())
- if not results: break
- yield results
- page += 1
-
-
if __name__ == '__main__':
main()
