| import logging |
| from getpass import getpass |
| from optparse import OptionParser |
| from pylons import c |
| import re |
| import os |
| from time import mktime |
| import time |
| import json |
| from urlparse import urlparse |
| import urllib2 |
| import urllib |
| from cookielib import CookieJar |
| from datetime import datetime |
| from ConfigParser import ConfigParser |
| import random |
| import string |
| |
| import sqlalchemy |
| from suds.client import Client |
| from ming.orm.ormsession import ThreadLocalORMSession |
| from ming.base import Object |
| |
| from allura import model as M |
| from allura.lib import helpers as h |
| from allura.lib import utils |
| |
| log = logging.getLogger('teamforge-import') |
| |
| ''' |
| |
| http://help.collab.net/index.jsp?topic=/teamforge520/reference/api-services.html |
| |
| http://www.open.collab.net/nonav/community/cif/csfe/50/javadoc/index.html?com/collabnet/ce/soap50/webservices/page/package-summary.html |
| |
| ''' |
| |
| options = None |
| s = None # security token |
| client = None # main api client |
| users = {} |
| |
| cj = CookieJar() |
| loggedInOpener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) |
| |
| def make_client(api_url, app): |
| return Client(api_url + app + '?wsdl', location=api_url + app) |
| |
| |
| def main(): |
| global options, s, client, users |
| defaults=dict( |
| api_url=None, |
| attachment_url='/sf/%s/do/%s/', |
| default_wiki_text='PRODUCT NAME HERE', |
| username=None, |
| password=None, |
| output_dir='teamforge-export/', |
| list_project_ids=False, |
| neighborhood=None, |
| neighborhood_shortname=None, |
| use_thread_import_id_when_reloading=False, |
| skip_wiki=False, |
| skip_frs_download=False, |
| skip_unsupported_check=False) |
| optparser = get_parser(defaults) |
| options, project_ids = optparser.parse_args() |
| if options.config_file: |
| config = ConfigParser() |
| config.read(options.config_file) |
| defaults.update( |
| (k, eval(v)) for k,v in config.items('teamforge-import')) |
| optparser = get_parser(defaults) |
| options, project_ids = optparser.parse_args() |
| |
| # neither specified, so do both |
| if not options.extract and not options.load: |
| options.extract = True |
| options.load = True |
| |
| if options.extract: |
| client = make_client(options.api_url, 'CollabNet') |
| api_v = client.service.getApiVersion() |
| if not api_v.startswith('5.4.'): |
| log.warning('Unexpected API Version %s. May not work correctly.' % api_v) |
| |
| s = client.service.login(options.username, options.password or getpass('Password: ')) |
| teamforge_v = client.service.getVersion(s) |
| if not teamforge_v.startswith('5.4.'): |
| log.warning('Unexpected TeamForge Version %s. May not work correctly.' % teamforge_v) |
| |
| if options.load: |
| if not options.neighborhood: |
| log.error('You must specify a neighborhood when loading') |
| return |
| try: |
| nbhd = M.Neighborhood.query.get(name=options.neighborhood) |
| except: |
| log.exception('error querying mongo') |
| log.error('This should be run as "paster script production.ini ../scripts/teamforge-import.py -- ...options.."') |
| return |
| assert nbhd |
| |
| if not project_ids: |
| if not options.extract: |
| log.error('You must specify project ids') |
| return |
| projects = client.service.getProjectList(s) |
| project_ids = [p.id for p in projects.dataRows] |
| |
| if options.list_project_ids: |
| print ' '.join(project_ids) |
| return |
| |
| if not os.path.exists(options.output_dir): |
| os.makedirs(options.output_dir) |
| for pid in project_ids: |
| if options.extract: |
| try: |
| project = client.service.getProjectData(s, pid) |
| log.info('Project: %s %s %s' % (project.id, project.title, project.path)) |
| out_dir = os.path.join(options.output_dir, project.id) |
| if not os.path.exists(out_dir): |
| os.mkdir(out_dir) |
| |
| get_project(project) |
| get_files(project) |
| if not options.skip_wiki: |
| get_homepage_wiki(project) |
| get_discussion(project) |
| get_news(project) |
| if not options.skip_unsupported_check: |
| check_unsupported_tools(project) |
| with open(os.path.join(options.output_dir, 'users.json'), 'w') as user_file: |
| json.dump(users, user_file, default=str) |
| except: |
| log.exception('Error extracting %s' % pid) |
| |
| if options.load: |
| try: |
| project = create_project(pid, nbhd) |
| except: |
| log.exception('Error creating %s' % pid) |
| |
| def load_users(): |
| ''' load the users data from file, if it hasn't been already ''' |
| global users |
| user_filename = os.path.join(options.output_dir, 'users.json') |
| if not users and os.path.exists(user_filename): |
| with open(user_filename) as user_file: |
| users = json.load(user_file, object_hook=Object) # Object for attribute access |
| |
| |
| def save_user(usernames): |
| if isinstance(usernames, basestring): |
| usernames = [usernames] |
| |
| load_users() |
| |
| for username in usernames: |
| if username not in users: |
| user_data = client.service.getUserData(s, username) |
| users[username] = Object(user_data) |
| if users[username].status != 'Active': |
| log.warn('user: %s status: %s' % (username, users[username].status)) |
| |
| |
| def get_project(project): |
| global client |
| cats = make_client(options.api_url, 'CategorizationApp') |
| |
| data = client.service.getProjectData(s, project.id) |
| access_level = { 1: 'public', 4: 'private', 3: 'gated community'}[ |
| client.service.getProjectAccessLevel(s, project.id) |
| ] |
| admins = client.service.listProjectAdmins(s, project.id).dataRows |
| members = client.service.getProjectMemberList(s, project.id).dataRows |
| groups = client.service.getProjectGroupList(s, project.id).dataRows |
| categories = cats.service.getProjectCategories(s, project.id).dataRows |
| save(json.dumps(dict( |
| data = dict(data), |
| access_level = access_level, |
| admins = map(dict, admins), |
| members = map(dict, members), |
| groups = map(dict, groups), |
| categories = map(dict, categories), |
| ), default=str), |
| project, project.id+'.json') |
| |
| if len(groups): |
| log.warn('Project has groups %s' % groups) |
| for u in admins: |
| if not u.status != 'active': |
| log.warn('inactive admin %s' % u) |
| if u.superUser: |
| log.warn('super user admin %s' % u) |
| |
| save_user(data.createdBy) |
| save_user(u.userName for u in admins) |
| save_user(u.userName for u in members) |
| |
| def get_user(orig_username): |
| 'returns an allura User object' |
| sf_username = make_valid_sf_username(orig_username) |
| |
| u = M.User.by_username(sf_username) |
| |
| if not u: |
| load_users() |
| user = users[orig_username] |
| if user.status != 'Active': |
| log.warn('Inactive user %s %s' % (orig_username, user.status)) |
| |
| if not 3 <= len(user.fullName) <= 32: |
| raise Exception('invalid fullName length: %s' % user.fullName) |
| if '@' not in user.email: |
| raise Exception('invalid email: %s' % user.email) |
| # FIXME: hardcoded SFX integration |
| from sfx.model import tables as T |
| nu = T.users.insert() |
| nu.execute(user_name=sf_username.encode('utf-8'), |
| email=user.email.lower().encode('utf-8'), |
| realname=user.fullName.encode('utf-8'), |
| status='A' if user.status == 'Active' else 'D', |
| language=275, # english trove id |
| timezone=user.timeZone, |
| user_pw=''.join(random.sample(string.printable, 32)), |
| unix_pw=''.join(random.sample(string.printable, 32)), |
| user_pw_modtime=int(time.time()), |
| mail_siteupdates=0, |
| add_date=int(time.time()), |
| ) |
| user_id = sqlalchemy.select([T.users.c.user_id], T.users.c.user_name==sf_username).execute().fetchone().user_id |
| npref = T.user_preferences.insert() |
| npref.execute(user_id=user_id, preference_name='country', preference_value='US') |
| npref.execute(user_id=user_id, preference_name='opt_research', preference_value=0) |
| npref.execute(user_id=user_id, preference_name='opt_thirdparty', preference_value=0) |
| |
| new_audit = T.audit_trail_user.insert() |
| new_audit.execute( |
| date=int(time.time()), |
| username='nobody', |
| ip_address='(imported)', |
| operation_resource=user_id, |
| operation='%s user account created by TeamForge import script' % user.status, |
| operation_target='', |
| ) |
| |
| u = M.User.by_username(sf_username) |
| assert u |
| return u |
| |
| def convert_project_shortname(teamforge_path): |
| 'convert from TeamForge to SF, and validate early' |
| tf_shortname = teamforge_path.split('.')[-1] |
| sf_shortname = tf_shortname.replace('_','-') |
| |
| # FIXME hardcoded translations |
| sf_shortname = { |
| 'i1': 'motorola-i1', |
| 'i9': 'motorola-i9', |
| 'devplatformforocap': 'ocap-dev-pltfrm', |
| 'sitewide': '--init--', |
| }.get(sf_shortname, sf_shortname) |
| |
| if not 3 <= len(sf_shortname) <= 15: |
| raise ValueError('Project name length must be between 3 & 15, inclusive: %s (%s)' % |
| (sf_shortname, len(sf_shortname))) |
| return sf_shortname |
| |
| |
| # FIXME hardcoded |
| skip_perms_usernames = set([ |
| 'faisal_saeed','dsarkisian','debonairamit','nishanthiremath','Bhuvnesh','bluetooth','cnkurzke','makow2','jannes1','Joel_Hegberg','Farroc','brian_chen','eirikur', |
| 'dmitry_flyorov','bipingm','MornayJo','ibv','b_weisshaar','k9srb','johnmmills','a_gomolitsky','filim','kapoor','ljzegers','jrukes','dwilson9','jlin','quickie', |
| 'johnbell','nnikolenko','Gaetan','Giannetta','Katia','jackhan','jacobwangus','adwankar','dinobrusco','qbarnes','ilmojung','clifford_chan','nbaig','fhutchi1', |
| 'rinofarina','baiyanbin','muralidhar','duanyiruo','bredding','mkolkey','manvith','nanduk','engyihan','deepsie','dabon','dino_jiang','mattrose','peter_j_wilhelm', |
| 'emx2500','jmcguire','lfilimowski','guruppandit','abhilashisme','edwinhm','rabbi','ferrans','guna','kevin_robinson','adathiruthi','kochen','onehap','kalanithi', |
| 'jamesn','obu001','chetanv','Avinash','HugoBoss','Han_Wei','mhooper','g16872','mfcarignano','jim_burke','kevin','arunkarra','adam_feng','pavan_scm','kostya_katz', |
| 'ppazderka','eileenzhuang','pyammine','judyho','ashoykh','rdemento','ibrahim','min_wang','arvind_setlur','moorthy_karthik','daniel_nelson','dms','esnmurthy', |
| 'rasa_bonyadlou','prashantjoshi','edkeating','billsaez','cambalindo','jims','bozkoyun','andry_deltsov','bpowers','manuel_milli','maryparsons','spriporov','yutianli', |
| 'xiebin','tnemeth1','udayaps','zzzzuser','timberger','sbarve1','zarman','rwallace67','thangavelu_arum','yuhuaixie','tingup','sekchai','sasanplus','rupal','sebastien_hertz', |
| 'sab8123','rony_lim','slava_kirillin','smwest','wendydu_yq','sco002','RonFred','spatnala','vd','Sunny','tthompson','sunijams','slaw','rodovich','zhangqingqi82','venki', |
| 'yuntaom','xiaojin','walterciocosta','straus','Thomas','stupka','wangyu','yaowang','wisekb','tyler_louie','smartgarfield','shekar_mahalingam', |
| 'venkata_akella','v_yellapragada','vavasthi','rpatel','zhengfang','sweetybala','vap','sergey','ymhuang','spatel78745' |
| ]) |
| |
| def create_project(pid, nbhd): |
| M.session.artifact_orm_session._get().skip_mod_date = True |
| data = loadjson(pid, pid+'.json') |
| #pprint(data) |
| log.info('Loading: %s %s %s' % (pid, data.data.title, data.data.path)) |
| shortname = convert_project_shortname(data.data.path) |
| |
| project = M.Project.query.get(shortname=shortname, neighborhood_id=nbhd._id) |
| if not project: |
| private = (data.access_level == 'private') |
| log.debug('Creating %s private=%s' % (shortname, private)) |
| one_admin = [u.userName for u in data.admins if u.status == 'Active'][0] |
| project = nbhd.register_project(shortname, |
| get_user(one_admin), |
| project_name=data.data.title, |
| private_project=private) |
| project.notifications_disabled = True |
| project.short_description = data.data.description |
| project.last_updated = datetime.strptime(data.data.lastModifiedDate, '%Y-%m-%d %H:%M:%S') |
| M.main_orm_session.flush(project) |
| # TODO: push last_updated to gutenberg? |
| # TODO: try to set createdDate? |
| |
| role_admin = M.ProjectRole.by_name('Admin', project) |
| admin_usernames = set() |
| for admin in data.admins: |
| # FIXME: skip non-active users |
| if admin.userName in skip_perms_usernames: |
| continue |
| admin_usernames.add(admin.userName) |
| user = get_user(admin.userName) |
| c.user = user |
| pr = user.project_role(project) |
| pr.roles = [ role_admin._id ] |
| ThreadLocalORMSession.flush_all() |
| role_developer = M.ProjectRole.by_name('Developer', project) |
| for member in data.members: |
| # FIXME: skip non-active users |
| if member.userName in skip_perms_usernames: |
| continue |
| if member.userName in admin_usernames: |
| continue |
| user = get_user(member.userName) |
| pr = user.project_role(project) |
| pr.roles = [ role_developer._id ] |
| ThreadLocalORMSession.flush_all() |
| project.labels = [cat.path.split('projects/categorization.root.')[1] for cat in data.categories] |
| icon_file = 'emsignia-MOBILITY-red.png' |
| if 'nsn' in project.labels or 'msi' in project.labels: |
| icon_file = 'emsignia-SOLUTIONS-blue.gif' |
| if project.icon: |
| M.ProjectFile.remove(dict(project_id=project._id, category='icon')) |
| with open(os.path.join('..','scripts',icon_file)) as fp: |
| M.ProjectFile.save_image( |
| icon_file, fp, content_type=utils.guess_mime_type(icon_file), |
| square=True, thumbnail_size=(48,48), |
| thumbnail_meta=dict(project_id=project._id,category='icon')) |
| ThreadLocalORMSession.flush_all() |
| |
| dirs = os.listdir(os.path.join(options.output_dir, pid)) |
| |
| frs_mapping = loadjson(pid, 'frs_mapping.json') |
| |
| if not options.skip_wiki and 'wiki' in dirs: |
| import_wiki(project, pid, nbhd) |
| if not options.skip_frs_download and not project.app_instance('downloads'): |
| project.install_app('Downloads', 'downloads') |
| if 'forum' in dirs: |
| import_discussion(project, pid, frs_mapping, shortname, nbhd) |
| if 'news' in dirs: |
| import_news(project, pid, frs_mapping, shortname, nbhd) |
| |
| project.notifications_disabled = False |
| ThreadLocalORMSession.flush_all() |
| return project |
| |
| def import_wiki(project, pid, nbhd): |
| from forgewiki import model as WM |
| def upload_attachments(page, pid, beginning): |
| dirpath = os.path.join(options.output_dir, pid, 'wiki', beginning) |
| if not os.path.exists(dirpath): return |
| files = os.listdir(dirpath) |
| for f in files: |
| with open(os.path.join(options.output_dir, pid, 'wiki', beginning, f)) as fp: |
| page.attach(f, fp, content_type=utils.guess_mime_type(f)) |
| pages = os.listdir(os.path.join(options.output_dir, pid, 'wiki')) |
| # handle the homepage content |
| if 'homepage_text.markdown' in pages: |
| home_app = project.app_instance('home') |
| h.set_context(project.shortname, 'home', neighborhood=nbhd) |
| # set permissions and config options |
| role_admin = M.ProjectRole.by_name('Admin')._id |
| role_anon = M.ProjectRole.by_name('*anonymous')._id |
| home_app.config.options['show_discussion'] = False |
| home_app.config.options['show_left_bar'] = False |
| home_app.config.options['show_right_bar'] = False |
| home_app.config.acl = [ |
| M.ACE.allow(role_anon, 'read'), |
| M.ACE.allow(role_admin, 'create'), |
| M.ACE.allow(role_admin, 'edit'), |
| M.ACE.allow(role_admin, 'delete'), |
| M.ACE.allow(role_admin, 'moderate'), |
| M.ACE.allow(role_admin, 'configure'), |
| M.ACE.allow(role_admin, 'admin')] |
| p = WM.Page.upsert('Home') |
| p.text = wiki2markdown(load(pid, 'wiki', 'homepage_text.markdown')) |
| upload_attachments(p, pid, 'homepage') |
| if 'HomePage.json' in pages and 'HomePage.markdown' in pages: |
| wiki_app = project.app_instance('wiki') |
| if not wiki_app: |
| wiki_app = project.install_app('Wiki', 'wiki') |
| h.set_context(project.shortname, 'wiki', neighborhood=nbhd) |
| # set permissions and config options |
| role_admin = M.ProjectRole.by_name('Admin')._id |
| role_anon = M.ProjectRole.by_name('*anonymous')._id |
| wiki_app.config.options['show_discussion'] = False |
| wiki_app.config.options['show_left_bar'] = False |
| wiki_app.config.options['show_right_bar'] = False |
| wiki_app.config.acl = [ |
| M.ACE.allow(role_anon, 'read'), |
| M.ACE.allow(role_admin, 'create'), |
| M.ACE.allow(role_admin, 'edit'), |
| M.ACE.allow(role_admin, 'delete'), |
| M.ACE.allow(role_admin, 'moderate'), |
| M.ACE.allow(role_admin, 'configure'), |
| M.ACE.allow(role_admin, 'admin')] |
| # make all the wiki pages |
| for page in pages: |
| ending = page[-5:] |
| beginning = page[:-5] |
| markdown_file = '%s.markdown' % beginning |
| if '.json' == ending and markdown_file in pages: |
| page_data = loadjson(pid, 'wiki', page) |
| content = load(pid, 'wiki', markdown_file) |
| if page == 'HomePage.json': |
| globals = WM.Globals.query.get(app_config_id=wiki_app.config._id) |
| if globals is not None: |
| globals.root = page_data.title |
| else: |
| globals = WM.Globals(app_config_id=wiki_app.config._id, root=page_data.title) |
| p = WM.Page.upsert(page_data.title) |
| p.viewable_by = ['all'] |
| p.text = wiki2markdown(content) |
| # upload attachments |
| upload_attachments(p, pid, beginning) |
| if not p.history().first(): |
| p.commit() |
| ThreadLocalORMSession.flush_all() |
| |
| def import_discussion(project, pid, frs_mapping, sf_project_shortname, nbhd): |
| from forgediscussion import model as DM |
| discuss_app = project.app_instance('discussion') |
| if not discuss_app: |
| discuss_app = project.install_app('Discussion', 'discussion') |
| h.set_context(project.shortname, 'discussion', neighborhood=nbhd) |
| assert c.app |
| # set permissions and config options |
| role_admin = M.ProjectRole.by_name('Admin')._id |
| role_developer = M.ProjectRole.by_name('Developer')._id |
| role_auth = M.ProjectRole.by_name('*authenticated')._id |
| role_anon = M.ProjectRole.by_name('*anonymous')._id |
| discuss_app.config.acl = [ |
| M.ACE.allow(role_anon, 'read'), |
| M.ACE.allow(role_auth, 'post'), |
| M.ACE.allow(role_auth, 'unmoderated_post'), |
| M.ACE.allow(role_developer, 'moderate'), |
| M.ACE.allow(role_admin, 'configure'), |
| M.ACE.allow(role_admin, 'admin')] |
| ThreadLocalORMSession.flush_all() |
| DM.Forum.query.remove(dict(app_config_id=discuss_app.config._id,shortname='general')) |
| forums = os.listdir(os.path.join(options.output_dir, pid, 'forum')) |
| for forum in forums: |
| ending = forum[-5:] |
| forum_name = forum[:-5] |
| if '.json' == ending and forum_name in forums: |
| forum_data = loadjson(pid, 'forum', forum) |
| fo = DM.Forum.query.get(shortname=forum_name, app_config_id=discuss_app.config._id) |
| if not fo: |
| fo = DM.Forum(app_config_id=discuss_app.config._id, shortname=forum_name) |
| fo.name = forum_data.title |
| fo.description = forum_data.description |
| fo_num_topics = 0 |
| fo_num_posts = 0 |
| topics = os.listdir(os.path.join(options.output_dir, pid, 'forum', forum_name)) |
| for topic in topics: |
| ending = topic[-5:] |
| topic_name = topic[:-5] |
| if '.json' == ending and topic_name in topics: |
| fo_num_topics += 1 |
| topic_data = loadjson(pid, 'forum', forum_name, topic) |
| thread_query = dict( |
| subject=topic_data.title, |
| discussion_id=fo._id, |
| app_config_id=discuss_app.config._id) |
| if not options.skip_thread_import_id_when_reloading: |
| # temporary/transitional. Just needed the first time |
| # running with this new code against an existing import |
| # that didn't have import_ids |
| thread_query['import_id'] = topic_data.id |
| to = DM.ForumThread.query.get(**thread_query) |
| if not to: |
| to = DM.ForumThread( |
| subject=topic_data.title, |
| discussion_id=fo._id, |
| import_id=topic_data.id, |
| app_config_id=discuss_app.config._id) |
| to.import_id=topic_data.id |
| to_num_replies = 0 |
| oldest_post = None |
| newest_post = None |
| posts = sorted(os.listdir(os.path.join(options.output_dir, pid, 'forum', forum_name, topic_name))) |
| for post in posts: |
| ending = post[-5:] |
| post_name = post[:-5] |
| if '.json' == ending: |
| to_num_replies += 1 |
| post_data = loadjson(pid, 'forum', forum_name, topic_name, post) |
| p = DM.ForumPost.query.get( |
| _id='%s%s@import' % (post_name,str(discuss_app.config._id)), |
| thread_id=to._id, |
| discussion_id=fo._id, |
| app_config_id=discuss_app.config._id) |
| |
| if not p: |
| p = DM.ForumPost( |
| _id='%s%s@import' % (post_name,str(discuss_app.config._id)), |
| thread_id=to._id, |
| discussion_id=fo._id, |
| app_config_id=discuss_app.config._id) |
| create_date = datetime.strptime(post_data.createdDate, '%Y-%m-%d %H:%M:%S') |
| p.timestamp = create_date |
| p.author_id = str(get_user(post_data.createdByUserName)._id) |
| p.text = convert_post_content(frs_mapping, sf_project_shortname, post_data.content, nbhd) |
| p.status = 'ok' |
| if post_data.replyToId: |
| p.parent_id = '%s%s@import' % (post_data.replyToId,str(discuss_app.config._id)) |
| slug, full_slug = p.make_slugs(parent = p.parent, timestamp = create_date) |
| p.slug = slug |
| p.full_slug = full_slug |
| if oldest_post == None or oldest_post.timestamp > create_date: |
| oldest_post = p |
| if newest_post == None or newest_post.timestamp < create_date: |
| newest_post = p |
| ThreadLocalORMSession.flush_all() |
| to.num_replies = to_num_replies |
| to.first_post_id = oldest_post._id |
| to.last_post_date = newest_post.timestamp |
| to.mod_date = newest_post.timestamp |
| fo_num_posts += to_num_replies |
| fo.num_topics = fo_num_topics |
| fo.num_posts = fo_num_posts |
| ThreadLocalORMSession.flush_all() |
| |
| def import_news(project, pid, frs_mapping, sf_project_shortname, nbhd): |
| from forgeblog import model as BM |
| posts = os.listdir(os.path.join(options.output_dir, pid, 'news')) |
| if len(posts): |
| news_app = project.app_instance('news') |
| if not news_app: |
| news_app = project.install_app('blog', 'news', mount_label='News') |
| h.set_context(project.shortname, 'news', neighborhood=nbhd) |
| # make all the blog posts |
| for post in posts: |
| if '.json' == post[-5:]: |
| post_data = loadjson(pid, 'news', post) |
| create_date = datetime.strptime(post_data.createdOn, '%Y-%m-%d %H:%M:%S') |
| p = BM.BlogPost.query.get(title=post_data.title, |
| timestamp=create_date, |
| app_config_id=news_app.config._id) |
| if not p: |
| p = BM.BlogPost(title=post_data.title, |
| timestamp=create_date, |
| app_config_id=news_app.config._id) |
| p.text = convert_post_content(frs_mapping, sf_project_shortname, post_data.body, nbhd) |
| p.mod_date = create_date |
| p.state = 'published' |
| if not p.slug: |
| p.make_slug() |
| if not p.history().first(): |
| p.commit() |
| ThreadLocalORMSession.flush_all() |
| M.Thread(discussion_id=p.app_config.discussion_id, |
| ref_id=p.index_id(), |
| subject='%s discussion' % p.title) |
| user = get_user(post_data.createdByUsername) |
| p.history().first().author=dict( |
| id=user._id, |
| username=user.username, |
| display_name=user.get_pref('display_name')) |
| ThreadLocalORMSession.flush_all() |
| |
| def check_unsupported_tools(project): |
| docs = make_client(options.api_url, 'DocumentApp') |
| doc_count = 0 |
| for doc in docs.service.getDocumentFolderList(s, project.id, recursive=True).dataRows: |
| if doc.title == 'Root Folder': |
| continue |
| doc_count += 1 |
| if doc_count: |
| log.warn('Migrating documents is not supported, but found %s docs' % doc_count) |
| |
| scm = make_client(options.api_url, 'ScmApp') |
| for repo in scm.service.getRepositoryList(s, project.id).dataRows: |
| log.warn('Migrating SCM repos is not supported, but found %s' % repo.repositoryPath) |
| |
| tasks = make_client(options.api_url, 'TaskApp') |
| task_count = len(tasks.service.getTaskList(s, project.id, filters=None).dataRows) |
| if task_count: |
| log.warn('Migrating tasks is not supported, but found %s tasks' % task_count) |
| |
| tracker = make_client(options.api_url, 'TrackerApp') |
| tracker_count = len(tracker.service.getArtifactList(s, project.id, filters=None).dataRows) |
| if tracker_count: |
| log.warn('Migrating trackers is not supported, but found %s tracker artifacts' % task_count) |
| |
| |
| def load(project_id, *paths): |
| in_file = os.path.join(options.output_dir, project_id, *paths) |
| with open(in_file) as input: |
| content = input.read() |
| return unicode(content, 'utf-8') |
| |
| def loadjson(*args): |
| # Object for attribute access |
| return json.loads(load(*args), object_hook=Object) |
| |
| def save(content, project, *paths): |
| out_file = os.path.join(options.output_dir, project.id, *paths) |
| if not os.path.exists(os.path.dirname(out_file)): |
| os.makedirs(os.path.dirname(out_file)) |
| with open(out_file, 'w') as out: |
| out.write(content.encode('utf-8')) |
| |
| def download_file(tool, url_path, *filepaths): |
| if tool == 'wiki': |
| action = 'viewAttachment' |
| elif tool == 'frs': |
| action = 'downloadFile' |
| else: |
| raise ValueError('tool %s not supported' % tool) |
| action_url = options.attachment_url % (tool, action) |
| |
| out_file = os.path.join(options.output_dir, *filepaths) |
| if not os.path.exists(os.path.dirname(out_file)): |
| os.makedirs(os.path.dirname(out_file)) |
| |
| if '://' in url_path: |
| url = url_path |
| else: |
| hostname = urlparse(options.api_url).hostname |
| scheme = urlparse(options.api_url).scheme |
| url = scheme + '://' + hostname + action_url + urllib.quote(url_path) |
| log.debug('fetching %s' % url) |
| |
| resp = loggedInOpener.open(url) |
| # if not logged in and this is private, you will get an html response instead of the file |
| # log in to make sure the file should really be html |
| if resp.headers.type == 'text/html': |
| # log in and save the file |
| resp = loggedInOpener.open(scheme + '://' + hostname + "/sf/sfmain/do/login", urllib.urlencode({ |
| 'username': options.username, |
| 'password': options.password, |
| 'returnToUrl': url, |
| 'sfsubmit': 'submit' |
| })) |
| with open(out_file, 'w') as out: |
| out.write(resp.fp.read()) |
| return out_file |
| |
| bracket_macro = re.compile(r'\[(.*?)\]') |
| h1 = re.compile(r'^!!!', re.MULTILINE) |
| h2 = re.compile(r'^!!', re.MULTILINE) |
| h3 = re.compile(r'^!', re.MULTILINE) |
| re_stats = re.compile(r'#+ .* [Ss]tatistics\n+(.*\[sf:.*?Statistics\].*)+') |
| def wiki2markdown(markup): |
| ''' |
| Partial implementation of http://help.collab.net/index.jsp?topic=/teamforge520/reference/wiki-wikisyntax.html |
| TODO: __ for bold |
| TODO: quote filenames with spaces, e.g. [[img src="foo bar.jpg"]] |
| ''' |
| def bracket_handler(matchobj): |
| snippet = matchobj.group(1) |
| ext = snippet.rsplit('.')[-1].lower() |
| # TODO: support [foo|bar.jpg] |
| if snippet.startswith('sf:'): |
| # can't handle these macros |
| return matchobj.group(0) |
| elif ext in ('jpg', 'gif', 'png'): |
| filename = snippet.split('/')[-1] |
| return '[[img src=%s]]' % filename |
| elif '|' in snippet: |
| text, link = snippet.split('|', 1) |
| return '[%s](%s)' % (text, link) |
| else: |
| # regular link |
| return '<%s>' % snippet |
| markup = bracket_macro.sub(bracket_handler, markup or '') |
| markup = h1.sub('#', markup) |
| markup = h2.sub('##', markup) |
| markup = h3.sub('###', markup) |
| |
| markup = re_stats.sub('', markup) |
| return markup |
| |
| re_rel = re.compile(r'\b(rel\d+)\b') |
| def convert_post_content(frs_mapping, sf_project_shortname, text, nbhd): |
| def rel_handler(matchobj): |
| relno = matchobj.group(1) |
| path = frs_mapping.get(relno) |
| if path: |
| return '<a href="/projects/%s.%s/files/%s">%s</a>' % ( |
| sf_project_shortname, nbhd.url_prefix.strip('/'), path, path) |
| else: |
| return relno |
| text = re_rel.sub(rel_handler, text or '') |
| return text |
| |
| |
| def find_image_references(markup): |
| 'yields filenames' |
| for matchobj in bracket_macro.finditer(markup): |
| snippet = matchobj.group(1) |
| ext = snippet.rsplit('.')[-1].lower() |
| if ext in ('jpg', 'gif', 'png'): |
| yield snippet |
| |
| def get_news(project): |
| ''' |
| Extracts news posts |
| ''' |
| app = make_client(options.api_url, 'NewsApp') |
| |
| # find the forums |
| posts = app.service.getNewsPostList(s, project.id) |
| for post in posts.dataRows: |
| save(json.dumps(dict(post), default=str), project, 'news', post.id+'.json') |
| save_user(post.createdByUsername) |
| |
| def get_discussion(project): |
| ''' |
| Extracts discussion forums and posts |
| ''' |
| app = make_client(options.api_url, 'DiscussionApp') |
| |
| # find the forums |
| forums = app.service.getForumList(s, project.id) |
| for forum in forums.dataRows: |
| forumname = forum.path.split('.')[-1] |
| log.info('Retrieving data for forum: %s' % forumname) |
| save(json.dumps(dict(forum), default=str), project, 'forum', forumname+'.json') |
| # topic in this forum |
| topics = app.service.getTopicList(s, forum.id) |
| for topic in topics.dataRows: |
| save(json.dumps(dict(topic), default=str), project, 'forum', forumname, topic.id+'.json') |
| # posts in this topic |
| posts = app.service.getPostList(s, topic.id) |
| for post in posts.dataRows: |
| save(json.dumps(dict(post), default=str), project, 'forum', forumname, topic.id, post.id+'.json') |
| save_user(post.createdByUserName) |
| |
| |
| def get_homepage_wiki(project): |
| ''' |
| Extracts home page and wiki pages |
| ''' |
| wiki = make_client(options.api_url, 'WikiApp') |
| |
| pages = {} |
| wiki_pages = wiki.service.getWikiPageList(s, project.id) |
| for wiki_page in wiki_pages.dataRows: |
| wiki_page = wiki.service.getWikiPageData(s, wiki_page.id) |
| pagename = wiki_page.path.split('/')[-1] |
| save(json.dumps(dict(wiki_page), default=str), project, 'wiki', pagename+'.json') |
| if not wiki_page.wikiText: |
| log.debug('skip blank wiki page %s' % wiki_page.path) |
| continue |
| pages[pagename] = wiki_page.wikiText |
| |
| # PageApp does not provide a useful way to determine the Project Home special wiki page |
| # so use some heuristics |
| homepage = None |
| if '$ProjectHome' in pages and options.default_wiki_text not in pages['$ProjectHome']: |
| homepage = pages.pop('$ProjectHome') |
| elif 'HomePage' in pages and options.default_wiki_text not in pages['HomePage']: |
| homepage = pages.pop('HomePage') |
| elif '$ProjectHome' in pages: |
| homepage = pages.pop('$ProjectHome') |
| elif 'HomePage' in pages: |
| homepage = pages.pop('HomePage') |
| else: |
| log.warn('did not find homepage') |
| |
| if homepage: |
| save(homepage, project, 'wiki', 'homepage_text.markdown') |
| for img_ref in find_image_references(homepage): |
| filename = img_ref.split('/')[-1] |
| if '://' in img_ref: |
| img_url = img_ref |
| else: |
| img_url = project.path + '/wiki/' + img_ref |
| download_file('wiki', img_url, project.id, 'wiki', 'homepage', filename) |
| |
| for path, text in pages.iteritems(): |
| if options.default_wiki_text in text: |
| log.debug('skipping default wiki page %s' % path) |
| else: |
| save(text, project, 'wiki', path+'.markdown') |
| for img_ref in find_image_references(text): |
| filename = img_ref.split('/')[-1] |
| if '://' in img_ref: |
| img_url = img_ref |
| else: |
| img_url = project.path + '/wiki/' + img_ref |
| download_file('wiki', img_url, project.id, 'wiki', path, filename) |
| |
| def _dir_sql(created_on, project, dir_name, rel_path): |
| assert options.neighborhood_shortname |
| if not rel_path: |
| parent_directory = "'1'" |
| else: |
| parent_directory = "(SELECT pfs_path FROM pfs_path WHERE path_name = '%s/')" % rel_path |
| sql = """ |
| UPDATE pfs |
| SET file_crtime = '%s' |
| WHERE source_pk = (SELECT project.project FROM project WHERE project.project_name = '%s.%s') |
| AND source_table = 'project' |
| AND pfs_type = 'd' |
| AND pfs_name = '%s' |
| AND parent_directory = %s; |
| """ % (created_on, convert_project_shortname(project.path), options.neighborhood_shortname, dir_name, parent_directory) |
| return sql |
| |
| def get_files(project): |
| frs = make_client(options.api_url, 'FrsApp') |
| valid_pfs_filename = re.compile(r'(?![. ])[-_ +.,=#~@!()\[\]a-zA-Z0-9]+(?<! )$') |
| pfs_output_dir = os.path.join(os.path.abspath(options.output_dir), 'PFS', convert_project_shortname(project.path)) |
| sql_updates = '' |
| |
| def handle_path(obj, prev_path): |
| path_component = obj.title.strip().replace('/', ' ').replace('&','').replace(':','') |
| path = os.path.join(prev_path, path_component) |
| if not valid_pfs_filename.match(path_component): |
| log.error('Invalid filename: "%s"' % path) |
| save(json.dumps(dict(obj), default=str), |
| project, 'frs', path+'.json') |
| return path |
| |
| frs_mapping = {} |
| |
| for pkg in frs.service.getPackageList(s, project.id).dataRows: |
| pkg_path = handle_path(pkg, '') |
| pkg_details = frs.service.getPackageData(s, pkg.id) # download count |
| save(json.dumps(dict(pkg_details), default=str), |
| project, 'frs', pkg_path+'_details.json') |
| |
| for rel in frs.service.getReleaseList(s, pkg.id).dataRows: |
| rel_path = handle_path(rel, pkg_path) |
| frs_mapping[rel['id']] = rel_path |
| rel_details = frs.service.getReleaseData(s, rel.id) # download count |
| save(json.dumps(dict(rel_details), default=str), |
| project, 'frs', rel_path+'_details.json') |
| |
| for file in frs.service.getFrsFileList(s, rel.id).dataRows: |
| details = frs.service.getFrsFileData(s, file.id) |
| |
| file_path = handle_path(file, rel_path) |
| save(json.dumps(dict(file, |
| lastModifiedBy=details.lastModifiedBy, |
| lastModifiedDate=details.lastModifiedDate, |
| ), |
| default=str), |
| project, |
| 'frs', |
| file_path+'.json' |
| ) |
| if not options.skip_frs_download: |
| download_file('frs', rel.path + '/' + file.id, pfs_output_dir, file_path) |
| mtime = int(mktime(details.lastModifiedDate.timetuple())) |
| os.utime(os.path.join(pfs_output_dir, file_path), (mtime, mtime)) |
| |
| # releases |
| created_on = int(mktime(rel.createdOn.timetuple())) |
| mtime = int(mktime(rel.lastModifiedOn.timetuple())) |
| if os.path.exists(os.path.join(pfs_output_dir, rel_path)): |
| os.utime(os.path.join(pfs_output_dir, rel_path), (mtime, mtime)) |
| sql_updates += _dir_sql(created_on, project, rel.title.strip(), pkg_path) |
| # packages |
| created_on = int(mktime(pkg.createdOn.timetuple())) |
| mtime = int(mktime(pkg.lastModifiedOn.timetuple())) |
| if os.path.exists(os.path.join(pfs_output_dir, pkg_path)): |
| os.utime(os.path.join(pfs_output_dir, pkg_path), (mtime, mtime)) |
| sql_updates += _dir_sql(created_on, project, pkg.title.strip(), '') |
| # save pfs update sql for this project |
| with open(os.path.join(options.output_dir, 'pfs_updates.sql'), 'a') as out: |
| out.write('/* %s */' % project.id) |
| out.write(sql_updates) |
| save(json.dumps(frs_mapping), project, 'frs_mapping.json') |
| |
| |
| def get_parser(defaults): |
| optparser = OptionParser( |
| usage=('%prog [--options] [projID projID projID]\n' |
| 'If no project ids are given, all projects will be migrated')) |
| optparser.set_defaults(**defaults) |
| |
| # Command-line-only options |
| optparser.add_option( |
| '--extract-only', action='store_true', dest='extract', |
| help='Store data from the TeamForge API on the local filesystem; not load into Allura') |
| optparser.add_option( |
| '--load-only', action='store_true', dest='load', |
| help='Load into Allura previously-extracted data') |
| optparser.add_option( |
| '--config-file', dest='config_file', |
| help='Load options from config file') |
| |
| # Command-line options with defaults in config file |
| optparser.add_option( |
| '--api-url', dest='api_url', help='e.g. https://hostname/ce-soap50/services/') |
| optparser.add_option( |
| '--attachment-url', dest='attachment_url') |
| optparser.add_option( |
| '--default-wiki-text', dest='default_wiki_text', |
| help='used in determining if a wiki page text is default or changed') |
| optparser.add_option( |
| '-u', '--username', dest='username') |
| optparser.add_option( |
| '-p', '--password', dest='password') |
| optparser.add_option( |
| '-o', '--output-dir', dest='output_dir') |
| optparser.add_option( |
| '--list-project-ids', action='store_true', dest='list_project_ids') |
| optparser.add_option( |
| '-n', '--neighborhood', dest='neighborhood', |
| help='Neighborhood full name, to load in to') |
| optparser.add_option( |
| '--n-shortname', dest='neighborhood_shortname', |
| help='Neighborhood shortname, for PFS extract SQL') |
| optparser.add_option( |
| '--skip-thread-import-id-when-reloading', action='store_true', |
| dest='skip_thread_import_id_when_reloading' |
| ) |
| optparser.add_option( |
| '--skip-frs-download', action='store_true', dest='skip_frs_download') |
| optparser.add_option( |
| '--skip-wiki', action='store_true', dest='skip_wiki') |
| optparser.add_option( |
| '--skip-unsupported-check', action='store_true', dest='skip_unsupported_check') |
| |
| return optparser |
| |
| re_username = re.compile(r"^[a-z\-0-9]+$") |
| def make_valid_sf_username(orig_username): |
| sf_username = orig_username.replace('_','-').lower() |
| |
| # FIXME username translation is hardcoded here: |
| sf_username = dict( |
| rlevy = 'ramilevy', |
| mkeisler = 'mkeisler', |
| bthale = 'bthale', |
| mmuller = 'mattjustmull', |
| MalcolmDwyer = 'slagheap', |
| tjyang = 'tjyang', |
| manaic = 'maniac76', |
| srinid = 'cnudav', |
| es = 'est016', |
| david_peyer = 'david-mmi', |
| okruse = 'ottokruse', |
| jvp = 'jvpmoto', |
| dmorelli = 'dmorelli', |
| ).get(sf_username, sf_username + '-mmi') |
| |
| if not re_username.match(sf_username): |
| adjusted_username = ''.join( |
| ch for ch in sf_username[:-4] |
| if ch.isalnum() or ch == '-') + '-mmi' |
| log.error('invalid sf_username characters: %s Changing it to %s', |
| sf_username, adjusted_username) |
| sf_username = adjusted_username |
| if len(sf_username) > 15: |
| adjusted_username = sf_username[0:15-4] + '-mmi' |
| log.error('invalid sf_username length: %s Changing it to %s', |
| sf_username, adjusted_username) |
| sf_username = adjusted_username |
| return sf_username |
| |
| if __name__ == '__main__': |
| logging.basicConfig(level=logging.WARN) |
| log.setLevel(logging.DEBUG) |
| main() |
| |
| def test_make_valid_sf_username(): |
| tests = { |
| # basic |
| 'foo':'foo-mmi', |
| # lookup |
| 'rlevy':'ramilevy', |
| # too long |
| 'u012345678901234567890': 'u0123456789-mmi', |
| 'foo^213': 'foo213-mmi' |
| } |
| for k,v in tests.iteritems(): |
| assert make_valid_sf_username(k) == v |
| |
| def test_convert_post_content(): |
| nbhd = Object() |
| nbhd.url_prefix = '/motorola/' |
| text = '''rel100? or ?rel101 or rel102 or rel103a or rel104''' |
| mapping = dict( |
| rel100='rel/100/', |
| rel101='rel/101/', |
| rel102='rel/102/', |
| rel103='rel/103/', |
| rel104='rel/104/') |
| converted = convert_post_content(mapping, 'foo', text, nbhd) |
| assert 'href="/projects/foo.motorola/files/rel/100' in converted, converted |
| assert 'href="/projects/foo.motorola/files/rel/101' in converted, converted |
| assert 'href="/projects/foo.motorola/files/rel/102' in converted, converted |
| assert 'href="/projects/foo.motorola/files/rel/103' not in converted, converted |
| assert 'href="/projects/foo.motorola/files/rel/104' in converted, converted |
| |
| |
| def test_convert_markup(): |
| |
| markup = ''' |
| !this is the first headline |
| Please note that this project is for distributing, discussing, and supporting the open source software we release. |
| |
| [http://www.google.com] |
| |
| [SourceForge |http://www.sf.net] |
| |
| [$ProjectHome/myimage.jpg] |
| [$ProjectHome/anotherimage.jpg] |
| |
| !!! Project Statistics |
| |
| |[sf:frsStatistics]|[sf:artifactStatistics]| |
| ''' |
| |
| new_markup = wiki2markdown(markup) |
| assert '\n[[img src=myimage.jpg]]\n[[img src=anotherimage.jpg]]\n' in new_markup |
| assert '\n###this is the first' in new_markup |
| assert '<http://www.google.com>' in new_markup |
| assert '[SourceForge ](http://www.sf.net)' in new_markup |
| assert '\n# Project Statistics' not in new_markup |
| assert '[sf:frsStatistics]' not in new_markup |
