| import os |
| import re |
| import json |
| import socket |
| |
| from buildbot.scheduler import AnyBranchScheduler |
| from buildbot.schedulers.basic import SingleBranchScheduler |
| from buildbot.schedulers.timed import Nightly |
| from buildbot.schedulers.forcesched import ForceScheduler |
| |
| from buildbot.process.factory import BuildFactory |
| from buildbot.config import BuilderConfig |
| |
| from buildbot.process.properties import renderer |
| from buildbot.process.properties import Interpolate as I |
| from buildbot.process.properties import Property as P |
| |
| from buildbot.changes.filter import ChangeFilter |
| |
| from buildbot.steps.source.git import Git |
| from buildbot.steps.transfer import FileDownload |
| from buildbot.steps.shell import ShellCommand, Test, SetPropertyFromCommand |
| from buildbot.steps.master import SetProperty |
| |
| from buildbot.status import words, results |
| |
| # config |
| MEDIC_CONFIG_FILE = os.path.join(FP, 'cordova-config.json') |
| REPOS_CONFIG_FILE = os.path.join(FP, 'cordova-repos.json') |
| |
| def parse_config_file(file_name): |
| with open(file_name, 'r') as config_file: |
| return json.load(config_file) |
| |
| medic_config = parse_config_file(MEDIC_CONFIG_FILE) |
| repos_config = parse_config_file(REPOS_CONFIG_FILE) |
| |
| # constants |
| BASE_WORKDIR = '.' |
| TEST_APP_NAME = 'mobilespec' |
| EXTRA_CONFIG_FILE_NAME = 'cordova-extra.conf' |
| NPM_CACHE_DIR_NAME = 'npm_cache' |
| NPM_TEMP_DIR_NAME = 'npm_tmp' |
| COUCHDB_URI = medic_config['couchdb']['uri'] |
| ENTRY_POINT = medic_config['app']['entry'] |
| TEST_RUN_TIMEOUT = medic_config['app']['timeout'] # in seconds |
| LOG_GETTING_TIMEOUT = 30 # in seconds |
| TEST_SUMMARY_FILE_NAME = 'test_summary.json' |
| MASTER_HOSTNAME = socket.gethostname() |
| GIT_RETRY_DELAY = 10 |
| GIT_RETRY_TIMES = 3 |
| |
| CORDOVA_SUPPORTED_CATEGORY = 'cordova' |
| CORDOVA_UNSUPPORTED_CATEGORY = 'cordova-medic-unsupported' |
| |
| CORE_PLUGINS = [ |
| 'cordova-plugins', |
| 'cordova-plugin-test-framework', |
| 'cordova-plugin-battery-status', |
| 'cordova-plugin-camera', |
| 'cordova-plugin-console', |
| 'cordova-plugin-contacts', |
| 'cordova-plugin-device', |
| 'cordova-plugin-device-motion', |
| 'cordova-plugin-device-orientation', |
| 'cordova-plugin-dialogs', |
| 'cordova-plugin-file', |
| 'cordova-plugin-file-transfer', |
| 'cordova-plugin-geolocation', |
| 'cordova-plugin-globalization', |
| 'cordova-plugin-inappbrowser', |
| 'cordova-plugin-media', |
| 'cordova-plugin-media-capture', |
| 'cordova-plugin-network-information', |
| 'cordova-plugin-splashscreen', |
| 'cordova-plugin-statusbar', |
| 'cordova-plugin-vibration', |
| 'cordova-plugin-whitelist', |
| ] |
| |
| # NOTE: |
| # this is a special value that must be '' in order for |
| # all other non-Cordova builders on a master to work; |
| # more info is in the Buildbot documentation: |
| # http://docs.buildbot.net/0.8.10/manual/concepts.html#codebase |
| # http://docs.buildbot.net/0.8.10/manual/concepts.html#multiple-codebase-builds |
| SPECIAL_DEFAULT_CODEBASE_NAME = '' |
| |
| # patterns |
| CORDOVA_REPO_PATTERN = r'.*(cordova-[^\.]+).*' |
| |
| ####### UTILITIES |
| |
| # custom steps |
| class DisplayResults(Test): |
| |
| def start(self): |
| |
| test_summary = json.loads(self.getProperty('test_summary')) |
| |
| total = test_summary['total'] |
| failed = test_summary['failed'] |
| passed = test_summary['passed'] |
| warnings = test_summary['warnings'] |
| |
| self.setTestResults(total=total, failed=failed, passed=passed, warnings=warnings) |
| self.finished(results.SUCCESS if failed == 0 else results.WARNINGS) |
| self.step_status.setText(self.describe(True)) |
| |
| # helper functions |
| def codebase_name_from_repo_uri(uri): |
| match = re.match(CORDOVA_REPO_PATTERN, uri) |
| if match is not None: |
| return match.group(1) |
| return SPECIAL_DEFAULT_CODEBASE_NAME |
| |
| def default_codebase_repo_uri(codebase_name): |
| codebase = repos_config[codebase_name] |
| all_repos = codebase['repositories'] |
| default_repo = codebase['default_repository'] |
| return all_repos[default_repo] |
| |
| def default_codebase_branch(codebase_name): |
| return repos_config[codebase_name]['default_branch'] |
| |
| def slugify(string): |
| return string.lower().replace(' ', '-') |
| |
| def make_codebase(codebase_name): |
| ''' |
| Return a dict of default values for the given codebase. |
| ''' |
| return { |
| 'repository': default_codebase_repo_uri(codebase_name), |
| 'branch': default_codebase_branch(codebase_name), |
| 'revision': None, |
| } |
| |
| def get_platform_codebase(platform): |
| ''' |
| Return the name of the codebase for the given platform. |
| This is almost always 'cordova-[PLATFORM]', except for |
| 'blackberry', where it's 'blackberry10'. |
| ''' |
| repo_name = 'cordova-{0}'.format(platform) |
| if platform == 'blackberry10': |
| repo_name = 'cordova-blackberry' |
| return repo_name |
| |
| # step wrappers |
| def DescribedStep(step_class, description, haltOnFailure=True, **kwargs): |
| return step_class(description=description, descriptionDone=description, name=slugify(description), haltOnFailure=haltOnFailure, **kwargs) |
| |
| def SH(workdir=BASE_WORKDIR, timeout=TEST_RUN_TIMEOUT, **kwargs): |
| return DescribedStep(ShellCommand, workdir=workdir, timeout=timeout, **kwargs) |
| |
| def NPM(npm_subcommand, command=list(), what='code', **kwargs): |
| return SH( |
| command = ['npm', npm_subcommand] + command, |
| description = 'npm ' + npm_subcommand + 'ing ' + what, |
| env = {'npm_config_prefix': P('npm_prefix_dir')}, |
| **kwargs |
| ) |
| |
| def NPMInstall(command=list(), **kwargs): |
| # NOTE: |
| # adding the --cache parameter so that we don't use the global |
| # npm cache, which is shared with other processes |
| # |
| # adding the --tmp parameter so that even if the command doesn't |
| # exit cleanly, the folder will get removed during cleanup; |
| # refer to: https://docs.npmjs.com/files/folders#temp-files |
| return NPM('install', command=command + ['--cache', P('npm_cache_dir'), '--tmp', P('npm_temp_dir')], **kwargs) |
| |
| def NPMTest(**kwargs): |
| return NPM('test', **kwargs) |
| |
| def NPMLink(**kwargs): |
| return NPM('link', **kwargs) |
| |
| def Clone(repourl, **kwargs): |
| return DescribedStep(Git, 'cloning', repourl=repourl, mode='full', method='clean', shallow=False, retryFetch=True, retry=(GIT_RETRY_DELAY, GIT_RETRY_TIMES), **kwargs) |
| |
| def CodebaseClone(codebase, *args, **kwargs): |
| return Clone(repourl=I('%(src:' + codebase + ':repository)s'), codebase=codebase, workdir=codebase, **kwargs) |
| |
| def Set(name, value, **kwargs): |
| return DescribedStep(SetProperty, 'setting ' + name, property=name, value=value, **kwargs) |
| |
| def SetFromCommand(name, command, **kwargs): |
| return DescribedStep(SetPropertyFromCommand, 'setting ' + name, property=name, command=command, **kwargs) |
| |
| def Download(mastersrc, slavedest, description, **kwargs): |
| # NOTE: |
| # the FileDownload step has a bug and requires |
| # the 'description' parameter to be a list |
| return FileDownload(mastersrc=mastersrc, slavedest=slavedest, description=[description], workdir=BASE_WORKDIR, **kwargs) |
| |
| ####### SLAVES |
| |
| # NOTE: |
| # these slave names refer to the ones specified in master.cfg, |
| # and they must remain defined in master.cfg in order to work |
| # with the master.cfg used on Apache's Buildbot |
| OSX_SLAVES = ['cordova-osx-slave'] |
| WINDOWS_SLAVES = ['cordova-windows-slave'] |
| ALL_CORDOVA_SLAVES = OSX_SLAVES + WINDOWS_SLAVES |
| |
| ####### CHANGESOURCES |
| |
| # None, because Apache Buildbot's master.cfg manages them, and since |
| # this file is shared with Apache Buildbot, we should not touch them. |
| |
| ####### CODEBASES |
| |
| def cordovaCG(change_dict): |
| return codebase_name_from_repo_uri(change_dict['repository']) |
| |
| def overrideCG(existingCG): |
| def newCG(change): |
| codebase = cordovaCG(change) |
| if codebase == SPECIAL_DEFAULT_CODEBASE_NAME: |
| codebase = existingCG(change) |
| return codebase |
| return newCG |
| |
| # NOTE: |
| # codebaseGenerator is a magic function required by Buildbot |
| # |
| # if a codebaseGenerator is already defined, we must only extend it; |
| # we choose to take precedence over the other implementation, only |
| # calling it if our own returns the SPECIAL_DEFAULT_CODEBASE_NAME codebase |
| if 'codebaseGenerator' in c: |
| codebaseGenerator = overrideCG(c['codebaseGenerator']) |
| else: |
| codebaseGenerator = cordovaCG |
| |
| c['codebaseGenerator'] = codebaseGenerator |
| |
| CORDOVA_CODEBASES = {codebase_name: make_codebase(codebase_name) for codebase_name in repos_config.keys()} |
| |
| ####### STEPS |
| |
| CORDOVA_STEPS_SET_SETTINGS = [ |
| |
| Set('build_id', I('%(prop:buildername)s-%(prop:buildnumber)s-' + MASTER_HOSTNAME)), |
| Set('test_summary_file', I('%(prop:builddir)s/' + TEST_SUMMARY_FILE_NAME)), |
| |
| Set('npm_cache_dir', I('%(prop:builddir)s/' + NPM_CACHE_DIR_NAME)), |
| Set('npm_temp_dir', I('%(prop:builddir)s/' + NPM_TEMP_DIR_NAME)), |
| Set('npm_prefix_dir', I('%(prop:builddir)s/')), |
| |
| SetFromCommand('npm_version', ['npm', '--version']), |
| SetFromCommand('node_version', ['node', '--version']), |
| |
| SH(command=['npm', 'ls', '-g'], description='getting global modules'), |
| ] |
| |
| CORDOVA_STEPS_CLEAN_UP = [ |
| SH(command=['node', 'cordova-medic/medic/medic.js', 'clean', '.', '--exclude', 'cordova-medic', '--exclude', NPM_CACHE_DIR_NAME], description='cleaning workspace'), |
| ] |
| |
| CORDOVA_STEPS_GET_MEDIC = [ |
| |
| CodebaseClone('cordova-medic'), |
| |
| # NOTE: |
| # --production switch is used to speed up installation + fruitstrap dev dependency is not supported on Windows |
| NPMInstall(command=['--production'], what='cordova-medic', workdir='cordova-medic'), |
| ] |
| |
| CORDOVA_STEPS_GET_TOOLS = [ |
| |
| # clone cordova CLI tools |
| CodebaseClone('cordova-cli'), |
| CodebaseClone('cordova-lib'), |
| CodebaseClone('cordova-js'), |
| CodebaseClone('cordova-plugman'), |
| |
| # install js |
| NPMInstall(workdir='cordova-js', what='cordova-js'), |
| NPMLink(workdir='cordova-js', what='cordova-js'), |
| |
| # install common lib code |
| NPMInstall(workdir='cordova-lib/cordova-common', what='cordova-common'), |
| NPMLink(workdir='cordova-lib/cordova-common', what='cordova-common'), |
| |
| # install lib |
| NPMLink(command=['cordova-js'], workdir='cordova-lib/cordova-lib', what='cloned cordova-js'), |
| NPMLink(command=['cordova-common'], workdir='cordova-lib/cordova-lib', what='cloned cordova-common'), |
| NPMInstall(workdir='cordova-lib/cordova-lib', what='cordova-lib'), |
| NPMLink(workdir='cordova-lib/cordova-lib', what='cordova-lib'), |
| |
| # install cli |
| NPMLink(command=['cordova-lib'], workdir='cordova-cli', what='cloned cordova-lib'), |
| NPMInstall(workdir='cordova-cli', what='cordova-cli'), |
| NPMLink(workdir='cordova-cli', what='cordova-cli'), |
| |
| # install plugman |
| NPMLink(command=['cordova-lib'], workdir='cordova-plugman', what='cloned cordova-lib'), |
| NPMInstall(workdir='cordova-plugman', what='cordova-plugman'), |
| |
| # clone and install other tools |
| CodebaseClone('cordova-coho'), |
| CodebaseClone('cordova-mobile-spec'), |
| |
| NPMInstall(workdir='cordova-coho', what='cordova-coho'), |
| NPMInstall(workdir='cordova-mobile-spec/createmobilespec', what='cordova-mobile-spec'), |
| ] |
| |
| def cordova_steps_get_plugins(plugins): |
| steps = [] |
| for plugin in plugins: |
| steps.append(CodebaseClone(plugin)) |
| return steps |
| |
| def cordova_steps_create_mobilespec(platform): |
| return [ |
| |
| # get and install platform |
| CodebaseClone(get_platform_codebase(platform)), |
| NPMInstall(what='platform', workdir=get_platform_codebase(platform)), |
| |
| # create mobilespec |
| SH( |
| command = [ |
| 'node', |
| 'cordova-mobile-spec/createmobilespec/createmobilespec.js', |
| '--copywww', |
| '--skiplink', |
| '--' + platform, |
| TEST_APP_NAME |
| ], |
| description='creating mobilespec app' |
| ), |
| ] |
| |
| def cordova_steps_run_tests(platform, extra_args=list()): |
| return [ |
| |
| # download medic's config to slave |
| Download(mastersrc=MEDIC_CONFIG_FILE, slavedest='cordova-medic/config.json', description='downloading master\'s config'), |
| |
| SH( |
| command = [ |
| 'node', |
| 'cordova-medic/medic/medic.js', |
| 'run', |
| '--id', P('build_id'), |
| '--platform', platform, |
| '--couchdb', COUCHDB_URI, |
| '--entry', ENTRY_POINT, |
| '--app', TEST_APP_NAME, |
| |
| # NOTE: |
| # this timeout is smaller because TEST_RUN_TIMEOUT is used as the |
| # buildbot timeout, and the "run" command needs to time out before |
| # the buildbot wrapper times out so it can exit cleanly on timeout |
| '--timeout', TEST_RUN_TIMEOUT - 60 |
| ] + extra_args, |
| description = 'running tests', |
| haltOnFailure = True, |
| ), |
| |
| SH( |
| command = [ |
| 'node', |
| 'cordova-medic/medic/medic.js', |
| 'log', |
| '--platform', platform, |
| '--app', TEST_APP_NAME, |
| '--timeout', TEST_RUN_TIMEOUT |
| ], |
| description = 'gathering logs', |
| timeout = LOG_GETTING_TIMEOUT, |
| haltOnFailure = False, |
| flunkOnFailure = False, |
| alwaysRun = True, |
| ), |
| |
| SH( |
| command = [ |
| 'node', |
| 'cordova-medic/medic/medic.js', |
| 'check', |
| '--id', P('build_id'), |
| '--couchdb', COUCHDB_URI, |
| '--file', P('test_summary_file'), |
| ], |
| description = 'getting test results', |
| haltOnFailure = True, |
| ), |
| |
| SetFromCommand('test_summary', ['cat', P('test_summary_file')], hideStepIf=True), |
| DisplayResults(warnOnWarnings=True), |
| ] |
| |
| def makeRunSteps(platform, extra_args=list()): |
| |
| factory = BuildFactory() |
| |
| factory.addSteps(CORDOVA_STEPS_SET_SETTINGS) |
| factory.addSteps(CORDOVA_STEPS_GET_MEDIC) |
| factory.addSteps([ |
| SH(command=['node', 'cordova-medic/medic/medic.js', 'kill', '--platform', platform], description='killing running tasks'), |
| ]) |
| factory.addSteps(CORDOVA_STEPS_CLEAN_UP) |
| factory.addSteps(CORDOVA_STEPS_GET_TOOLS) |
| |
| factory.addSteps(cordova_steps_get_plugins(CORE_PLUGINS)) |
| factory.addSteps(cordova_steps_create_mobilespec(platform)) |
| factory.addSteps(cordova_steps_run_tests(platform, extra_args=extra_args)) |
| factory.addSteps([ |
| SH(command=['node', 'cordova-medic/medic/medic.js', 'kill', '--platform', platform], description='killing running tasks'), |
| ]) |
| |
| return factory |
| |
| ####### BUILDERS |
| |
| cordova_run_android = makeRunSteps('android') |
| cordova_run_ios = makeRunSteps('ios') |
| cordova_run_ws81 = makeRunSteps('windows', extra_args=['--winvers', 'store']) |
| cordova_run_wp81 = makeRunSteps('windows', extra_args=['--winvers', 'phone']) |
| # cordova_run_bbry = makeRunSteps('blackberry10') |
| |
| c['builders'].extend([ |
| |
| BuilderConfig(name='cordova-android-osx', slavenames=OSX_SLAVES, factory=cordova_run_android, category=CORDOVA_SUPPORTED_CATEGORY), |
| BuilderConfig(name='cordova-android-win', slavenames=WINDOWS_SLAVES, factory=cordova_run_android, category=CORDOVA_SUPPORTED_CATEGORY), |
| BuilderConfig(name='cordova-ios', slavenames=OSX_SLAVES, factory=cordova_run_ios, category=CORDOVA_SUPPORTED_CATEGORY), |
| BuilderConfig(name='cordova-windows-store8.1', slavenames=WINDOWS_SLAVES, factory=cordova_run_ws81, category=CORDOVA_SUPPORTED_CATEGORY), |
| BuilderConfig(name='cordova-windows-phone8.1', slavenames=WINDOWS_SLAVES, factory=cordova_run_wp81, category=CORDOVA_SUPPORTED_CATEGORY), |
| # BuilderConfig(name='cordova-blackberry-win', slavenames=WINDOWS_SLAVES, factory=cordova_run_bbry, category=CORDOVA_UNSUPPORTED_CATEGORY), |
| # BuilderConfig(name='cordova-blackberry-osx', slavenames=OSX_SLAVES, factory=cordova_run_bbry, category=CORDOVA_UNSUPPORTED_CATEGORY), |
| ]) |
| |
| ####### STATUS TARGETS |
| |
| c['status'].extend([]) |
| |
| ####### CHANGE FILTERS |
| |
| cordova_e2e_run_triggers = ChangeFilter(codebase=CORE_PLUGINS + [ |
| 'cordova-cli', |
| 'cordova-js', |
| 'cordova-lib', |
| 'cordova-plugman', |
| 'cordova-mobile-spec', |
| 'cordova-medic', |
| 'cordova-windows', |
| 'cordova-android', |
| 'cordova-ios', |
| # 'cordova-blackberry', |
| ]) |
| |
| ####### SCHEDULERS |
| |
| c['schedulers'].extend([ |
| |
| Nightly( |
| name = 'cordova-e2e-periodic', |
| reason = 'periodic', |
| codebases = CORDOVA_CODEBASES, |
| branch = None, # None means default branch |
| hour = [0, 6, 12, 18], |
| minute = 0, |
| onlyIfChanged = False, |
| builderNames = [ |
| 'cordova-android-osx', |
| 'cordova-android-win', |
| 'cordova-ios', |
| 'cordova-windows-store8.1', |
| 'cordova-windows-phone8.1', |
| # 'cordova-blackberry-win', |
| # 'cordova-blackberry-osx', |
| ], |
| ), |
| |
| SingleBranchScheduler( |
| name = 'cordova-e2e-commit', |
| reason = 'commit', |
| |
| # NOTE: |
| # this means that the scheduler will use source stamps |
| # that it saw when the build was triggered; otherwise, builds that |
| # build "master" might build different code if master changes |
| # between two builders from a set of builds that should be identical |
| createAbsoluteSourceStamps = True, |
| |
| # NOTE: |
| # in addition to 'change_filter', the 'codebases' argument ALSO functions |
| # as a filter, and codebases not present in it will not trigger the scheduler |
| change_filter = cordova_e2e_run_triggers, |
| codebases = CORDOVA_CODEBASES, |
| |
| builderNames = [ |
| 'cordova-android-osx', |
| 'cordova-android-win', |
| 'cordova-ios', |
| 'cordova-windows-store8.1', |
| 'cordova-windows-phone8.1', |
| # 'cordova-blackberry-win', |
| # 'cordova-blackberry-osx', |
| ], |
| ), |
| |
| ForceScheduler( |
| name = 'cordova-e2e-force', |
| codebases = CORDOVA_CODEBASES, |
| builderNames = [ |
| 'cordova-android-osx', |
| 'cordova-android-win', |
| 'cordova-ios', |
| 'cordova-windows-store8.1', |
| 'cordova-windows-phone8.1', |
| # 'cordova-blackberry-win', |
| # 'cordova-blackberry-osx', |
| ], |
| ), |
| ]) |
| |
| ####### EXTRA CONFIG |
| |
| # run the extra config file as if it was pasted |
| # below, passing it a copy of our globals |
| extra_config_path = os.path.join(FP, EXTRA_CONFIG_FILE_NAME) |
| if os.path.exists(extra_config_path): |
| print 'Loading extra Cordova config' |
| globals_copy = globals().copy() |
| execfile(extra_config_path, globals_copy, globals_copy) |
| else: |
| print 'No extra Cordova config found' |
