| #!/usr/bin/env python |
| |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| """Usage: benchmark.py run|list|compare|show|chart <selection> ... |
| |
| SELECTING TIMINGS -- B@R,LxS |
| |
| In the subcommands below, a timings selection consists of a string with up to |
| four elements: |
| <branch>@<revision>,<levels>x<spread> |
| abbreviated as: |
| B@R,LxS |
| |
| <branch> is a label of an svn branch, e.g. "1.7.x". |
| <revision> is the last-changed-revision of above branch. |
| <levels> is the number of directory levels created in the benchmark. |
| <spread> is the number of child trees spreading off each dir level. |
| |
| <branch_name> and <revision> are simply used for labeling. Upon the actual |
| test runs, you should enter labels matching the selected --svn-bin-dir. |
| Later, you can select runs individually by using these labels. |
| |
| For <revision>, you can provide special keywords: |
| - 'each' has the same effect as entering each available revision number that |
| is on record in the db in a separate timings selection. |
| - 'last' is the same as 'each', but shows only the last 10 revisions. 'last' |
| can be combined with a number, e.g. 'last12'. |
| |
| For all subcommands except 'run', you can omit some or all of the elements of |
| a timings selection to combine all available timings sets. Try that out with |
| the 'list' subcommand. |
| |
| Examples: |
| benchmark.py run 1.7.x@12345,5x5 |
| benchmark.py show trunk@12345 |
| benchmark.py compare 1.7.0,1x100 trunk@each,1x100 |
| benchmark.py chart compare 1.7.0,5x5 trunk@last12,5x5 |
| |
| |
| RUN BENCHMARKS |
| |
| benchmark.py run B@R,LxS [N] [options] |
| |
| Test data is added to an sqlite database created automatically, by default |
| 'benchmark.db' in the current working directory. To specify a different path, |
| use option -f <path_to_db>. |
| |
| If <N> is provided, the run is repeated N times. |
| |
| <levels> and <spread> control the way the tested working copy is structured: |
| <levels>: number of directory levels to create. |
| <spread>: number of files and subdirectories created in each dir. |
| |
| |
| LIST WHAT IS ON RECORD |
| |
| benchmark.py list [B@R,LxS] |
| |
| Find entries in the database for the given constraints. Any arguments can |
| be omitted. (To select only a rev, start with a '@', like '@123'; to select |
| only spread, start with an 'x', like "x100".) |
| |
| Call without arguments to get a listing of all available constraints. |
| |
| |
| COMPARE TIMINGS |
| |
| benchmark.py compare B@R,LxS B@R,LxS [B@R,LxS [...]] |
| |
| Compare any number of timings sets to the first provided set (in text mode). |
| For example: |
| benchmark.py compare 1.7.0 trunk@1349903 |
| Compare the total timings of all combined '1.7.0' branch runs to |
| all combined runs of 'trunk'-at-revision-1349903. |
| benchmark.py compare 1.7.0,5x5 trunk@1349903,5x5 |
| Same as above, but only compare the working copy types with 5 levels |
| and a spread of 5. |
| |
| Use the -c option to limit comparison to specific command names. |
| |
| |
| SHOW TIMINGS |
| |
| benchmark.py show B@R,LxS [B@R,LxS [...]] |
| |
| Print out a summary of the timings selected from the given constraints. |
| |
| |
| GENERATE CHARTS |
| |
| benchmark.py chart compare B@R,LxS B@R,LxS [ B@R,LxS ... ] |
| |
| Produce a bar chart that compares any number of sets of timings. Like with |
| the plain 'compare' command, the first set is taken as a reference point for |
| 100% and +-0 seconds. Each following dataset produces a set of labeled bar |
| charts, grouped by svn command names. At least two timings sets must be |
| provided. |
| |
| Use the -c option to limit comparison to specific command names. |
| |
| |
| EXAMPLES |
| |
| # Run 3 benchmarks on svn 1.7.0 with 5 dir levels and 5 files and subdirs for |
| # each level (spread). Timings are saved in ./benchmark.db. |
| # Provide label '1.7.0' and its Last-Changed-Rev for later reference. |
| ./benchmark.py run --svn-bin-dir ~/svn-prefix/1.7.0/bin 1.7.0@1181106,5x5 3 |
| |
| # Record 3 benchmark runs on trunk, again naming its Last-Changed-Rev. |
| # (You may also set your $PATH instead of using --svn-bin-dir.) |
| ./benchmark.py run --svn-bin-dir ~/svn-prefix/trunk/bin trunk@1352725,5x5 3 |
| |
| # Work with the results of above two runs |
| ./benchmark.py list |
| ./benchmark.py compare 1.7.0 trunk |
| ./benchmark.py show 1.7.0 trunk |
| ./benchmark.py chart compare 1.7.0 trunk |
| ./benchmark.py chart compare 1.7.0 trunk -c "update,commit,TOTAL RUN" |
| |
| # Rebuild r1352598, run it and chart improvements since 1.7.0. |
| svn up -r1352598 ~/src/trunk |
| make -C ~/src/trunk dist-clean install |
| export PATH="$HOME/svn-prefix/trunk/bin:$PATH" |
| which svn |
| ./benchmark.py run trunk@1352598,5x5 3 |
| ./benchmark.py chart compare 1.7.0 trunk@1352598 trunk@1352725 -o chart.svg |
| |
| |
| GLOBAL OPTIONS""" |
| |
| import os |
| import time |
| import datetime |
| import sqlite3 |
| import optparse |
| import tempfile |
| import subprocess |
| import random |
| import shutil |
| import stat |
| import string |
| from copy import copy |
| |
| IGNORE_COMMANDS = ('--version', ) |
| TOTAL_RUN = 'TOTAL RUN' |
| |
| j = os.path.join |
| |
| def bail(msg=None): |
| if msg: |
| print msg |
| exit(1) |
| |
| def time_str(): |
| return time.strftime('%Y-%m-%d %H:%M:%S'); |
| |
| def timedelta_to_seconds(td): |
| return ( float(td.seconds) |
| + float(td.microseconds) / (10**6) |
| + td.days * 24 * 60 * 60 ) |
| |
| def run_cmd(cmd, stdin=None, shell=False, verbose=False): |
| if options.verbose: |
| if shell: |
| printable_cmd = cmd |
| else: |
| printable_cmd = ' '.join(cmd) |
| print 'CMD:', printable_cmd |
| |
| if stdin: |
| stdin_arg = subprocess.PIPE |
| else: |
| stdin_arg = None |
| |
| p = subprocess.Popen(cmd, |
| stdin=stdin_arg, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| shell=shell) |
| stdout,stderr = p.communicate(input=stdin) |
| |
| if verbose: |
| if (stdout): |
| print "STDOUT: [[[\n%s]]]" % ''.join(stdout) |
| if (stderr): |
| print "STDERR: [[[\n%s]]]" % ''.join(stderr) |
| |
| return stdout, stderr |
| |
| |
| _next_unique_basename_count = 0 |
| |
| def next_unique_basename(prefix): |
| global _next_unique_basename_count |
| _next_unique_basename_count += 1 |
| return '_'.join((prefix, str(_next_unique_basename_count))) |
| |
| |
| si_units = [ |
| (1000 ** 5, 'P'), |
| (1000 ** 4, 'T'), |
| (1000 ** 3, 'G'), |
| (1000 ** 2, 'M'), |
| (1000 ** 1, 'K'), |
| (1000 ** 0, ''), |
| ] |
| def n_label(n): |
| """(stolen from hurry.filesize)""" |
| for factor, suffix in si_units: |
| if n >= factor: |
| break |
| amount = int(n/factor) |
| if isinstance(suffix, tuple): |
| singular, multiple = suffix |
| if amount == 1: |
| suffix = singular |
| else: |
| suffix = multiple |
| return str(amount) + suffix |
| |
| |
| def split_arg_once(l_r, sep): |
| if not l_r: |
| return (None, None) |
| if sep in l_r: |
| l, r = l_r.split(sep) |
| else: |
| l = l_r |
| r = None |
| if not l: |
| l = None |
| if not r: |
| r = None |
| return (l, r) |
| |
| RUN_KIND_SEPARATORS=('@', ',', 'x') |
| |
| class RunKind: |
| def __init__(self, b_r_l_s): |
| b_r, l_s = split_arg_once(b_r_l_s, RUN_KIND_SEPARATORS[1]) |
| self.branch, self.revision = split_arg_once(b_r, RUN_KIND_SEPARATORS[0]) |
| self.levels, self.spread = split_arg_once(l_s, RUN_KIND_SEPARATORS[2]) |
| if self.levels: self.levels = int(self.levels) |
| if self.spread: self.spread = int(self.spread) |
| |
| def label(self): |
| label_parts = [] |
| if self.branch: |
| label_parts.append(self.branch) |
| if self.revision: |
| label_parts.append(RUN_KIND_SEPARATORS[0]) |
| label_parts.append(self.revision) |
| if self.levels or self.spread: |
| label_parts.append(RUN_KIND_SEPARATORS[1]) |
| if self.levels: |
| label_parts.append(str(self.levels)) |
| if self.spread: |
| label_parts.append(RUN_KIND_SEPARATORS[2]) |
| label_parts.append(str(self.spread)) |
| return ''.join(label_parts) |
| |
| def args(self): |
| return (self.branch, self.revision, self.levels, self.spread) |
| |
| |
| def parse_timings_selections(db, *args): |
| run_kinds = [] |
| |
| for arg in args: |
| run_kind = RunKind(arg) |
| |
| if run_kind.revision == 'each': |
| run_kind.revision = None |
| query = TimingQuery(db, run_kind) |
| for revision in query.get_sorted_revisions(): |
| revision_run_kind = copy(run_kind) |
| revision_run_kind.revision = revision |
| run_kinds.append(revision_run_kind) |
| elif run_kind.revision and run_kind.revision.startswith('last'): |
| Nstr = run_kind.revision[4:] |
| if not Nstr: |
| N = 10 |
| else: |
| N = int(Nstr) |
| run_kind.revision = None |
| query = TimingQuery(db, run_kind) |
| for revision in query.get_sorted_revisions()[-N:]: |
| revision_run_kind = copy(run_kind) |
| revision_run_kind.revision = revision |
| run_kinds.append(revision_run_kind) |
| else: |
| run_kinds.append(run_kind) |
| |
| return run_kinds |
| |
| def parse_one_timing_selection(db, *args): |
| run_kinds = parse_timings_selections(db, *args) |
| if len(run_kinds) != 1: |
| bail("I need exactly one timings identifier, not '%s'" |
| % (' '.join(*args))) |
| return run_kinds[0] |
| |
| |
| |
| |
| PATHNAME_VALID_CHARS = "-_.,@%s%s" % (string.ascii_letters, string.digits) |
| def filesystem_safe_string(s): |
| return ''.join(c for c in s if c in PATHNAME_VALID_CHARS) |
| |
| def do_div(ref, val): |
| if ref: |
| return float(val) / float(ref) |
| else: |
| return 0.0 |
| |
| def do_diff(ref, val): |
| return float(val) - float(ref) |
| |
| |
| # ------------------------- database ------------------------- |
| |
| class TimingsDb: |
| def __init__(self, db_path): |
| self.db_path = db_path; |
| self.conn = sqlite3.connect(db_path) |
| self.ensure_tables_created() |
| |
| def ensure_tables_created(self): |
| c = self.conn.cursor() |
| |
| c.execute("""SELECT name FROM sqlite_master WHERE type='table' AND |
| name='batch'""") |
| if c.fetchone(): |
| # exists |
| return |
| |
| print 'Creating database tables.' |
| c.executescript(''' |
| CREATE TABLE batch ( |
| batch_id INTEGER PRIMARY KEY AUTOINCREMENT, |
| started TEXT, |
| ended TEXT |
| ); |
| |
| CREATE TABLE run_kind ( |
| run_kind_id INTEGER PRIMARY KEY AUTOINCREMENT, |
| branch TEXT NOT NULL, |
| revision TEXT NOT NULL, |
| wc_levels INTEGER, |
| wc_spread INTEGER, |
| UNIQUE(branch, revision, wc_levels, wc_spread) |
| ); |
| |
| CREATE TABLE run ( |
| run_id INTEGER PRIMARY KEY AUTOINCREMENT, |
| batch_id INTEGER NOT NULL REFERENCES batch(batch_id), |
| run_kind_id INTEGER NOT NULL REFERENCES run_kind(run_kind_id), |
| started TEXT, |
| ended TEXT, |
| aborted INTEGER |
| ); |
| |
| CREATE TABLE timings ( |
| run_id INTEGER NOT NULL REFERENCES run(run_id), |
| command TEXT NOT NULL, |
| sequence INTEGER, |
| timing REAL |
| );''' |
| ) |
| self.conn.commit() |
| c.close(); |
| |
| |
| class Batch: |
| def __init__(self, db): |
| self.db = db |
| self.started = time_str() |
| c = db.conn.cursor() |
| c.execute("INSERT INTO batch (started) values (?)", (self.started,)) |
| db.conn.commit() |
| self.id = c.lastrowid |
| c.close() |
| |
| def done(self): |
| conn = self.db.conn |
| c = conn.cursor() |
| c.execute(""" |
| UPDATE batch |
| SET ended = ? |
| WHERE batch_id = ?""", |
| (time_str(), self.id)) |
| conn.commit() |
| c.close() |
| |
| class Run: |
| def __init__(self, batch, run_kind): |
| self.batch = batch |
| conn = self.batch.db.conn |
| c = conn.cursor() |
| |
| c.execute(""" |
| SELECT run_kind_id FROM run_kind |
| WHERE branch = ? |
| AND revision = ? |
| AND wc_levels = ? |
| AND wc_spread = ?""", |
| run_kind.args()) |
| kind_ids = c.fetchone() |
| if kind_ids: |
| kind_id = kind_ids[0] |
| else: |
| c.execute(""" |
| INSERT INTO run_kind (branch, revision, wc_levels, wc_spread) |
| VALUES (?, ?, ?, ?)""", |
| run_kind.args()) |
| conn.commit() |
| kind_id = c.lastrowid |
| |
| self.started = time_str() |
| |
| c.execute(""" |
| INSERT INTO run |
| (batch_id, run_kind_id, started) |
| VALUES |
| (?, ?, ?)""", |
| (self.batch.id, kind_id, self.started)) |
| conn.commit() |
| self.id = c.lastrowid |
| c.close(); |
| self.tic_at = None |
| self.current_command = None |
| self.timings = [] |
| |
| def tic(self, command): |
| if command in IGNORE_COMMANDS: |
| return |
| self.toc() |
| self.current_command = command |
| self.tic_at = datetime.datetime.now() |
| |
| def toc(self): |
| if self.current_command and self.tic_at: |
| toc_at = datetime.datetime.now() |
| self.remember_timing(self.current_command, |
| timedelta_to_seconds(toc_at - self.tic_at)) |
| self.current_command = None |
| self.tic_at = None |
| |
| def remember_timing(self, command, seconds): |
| self.timings.append((command, seconds)) |
| |
| def submit_timings(self): |
| conn = self.batch.db.conn |
| c = conn.cursor() |
| print 'submitting...' |
| |
| c.executemany(""" |
| INSERT INTO timings |
| (run_id, command, sequence, timing) |
| VALUES |
| (?, ?, ?, ?)""", |
| [(self.id, t[0], (i + 1), t[1]) for i,t in enumerate(self.timings)]) |
| |
| conn.commit() |
| c.close() |
| |
| def done(self, aborted=False): |
| conn = self.batch.db.conn |
| c = conn.cursor() |
| c.execute(""" |
| UPDATE run |
| SET ended = ?, aborted = ? |
| WHERE run_id = ?""", |
| (time_str(), aborted, self.id)) |
| conn.commit() |
| c.close() |
| |
| |
| class TimingQuery: |
| def __init__(self, db, run_kind): |
| self.cursor = db.conn.cursor() |
| self.constraints = [] |
| self.values = [] |
| self.timings = None |
| self.FROM_WHERE = """ |
| FROM batch AS b, |
| timings AS t, |
| run AS r, |
| run_kind as k |
| WHERE |
| t.run_id = r.run_id |
| AND k.run_kind_id = r.run_kind_id |
| AND b.batch_id = r.batch_id |
| AND r.aborted = 0 |
| """ |
| self.append_constraint('k.branch', run_kind.branch) |
| self.each_revision = False |
| if run_kind.revision == 'each': |
| self.each_revision = True |
| else: |
| self.append_constraint('k.revision', run_kind.revision) |
| self.append_constraint('k.wc_levels', run_kind.levels) |
| self.append_constraint('k.wc_spread', run_kind.spread) |
| self.label = run_kind.label() |
| |
| def append_constraint(self, column_name, val): |
| if val: |
| self.constraints.append('AND %s = ?' % column_name) |
| self.values.append(val) |
| |
| def remove_last_constraint(self): |
| del self.constraints[-1] |
| del self.values[-1] |
| |
| def get_sorted_X(self, x, n=1): |
| query = ['SELECT DISTINCT %s' % x, |
| self.FROM_WHERE ] |
| query.extend(self.constraints) |
| query.append('ORDER BY %s' % x) |
| c = db.conn.cursor() |
| try: |
| c.execute(' '.join(query), self.values) |
| if n == 1: |
| return [tpl[0] for tpl in c.fetchall()] |
| else: |
| return c.fetchall() |
| finally: |
| c.close() |
| |
| def get_sorted_command_names(self): |
| return self.get_sorted_X('t.command') |
| |
| def get_sorted_branches(self): |
| return self.get_sorted_X('k.branch') |
| |
| def get_sorted_revisions(self): |
| return self.get_sorted_X('k.revision') |
| |
| def get_sorted_levels_spread(self): |
| return self.get_sorted_X('k.wc_levels,k.wc_spread', n = 2) |
| |
| def count_runs_batches(self): |
| query = ["""SELECT |
| count(DISTINCT r.run_id), |
| count(DISTINCT b.batch_id)""", |
| self.FROM_WHERE ] |
| query.extend(self.constraints) |
| c = db.conn.cursor() |
| try: |
| #print ' '.join(query) |
| c.execute(' '.join(query), self.values) |
| return c.fetchone() |
| finally: |
| c.close() |
| |
| def get_command_timings(self, command): |
| query = ["""SELECT |
| count(t.timing), |
| min(t.timing), |
| max(t.timing), |
| avg(t.timing)""", |
| self.FROM_WHERE ] |
| self.append_constraint('t.command', command) |
| try: |
| query.extend(self.constraints) |
| c = db.conn.cursor() |
| try: |
| c.execute(' '.join(query), self.values) |
| return c.fetchone() |
| finally: |
| c.close() |
| finally: |
| self.remove_last_constraint() |
| |
| def get_timings(self): |
| if self.timings: |
| return self.timings |
| self.timings = {} |
| for command_name in self.get_sorted_command_names(): |
| self.timings[command_name] = self.get_command_timings(command_name) |
| return self.timings |
| |
| |
| # ------------------------------------------------------------ run tests |
| |
| |
| def perform_run(batch, run_kind, |
| svn_bin, svnadmin_bin, verbose): |
| |
| run = Run(batch, run_kind) |
| |
| def create_tree(in_dir, _levels, _spread): |
| try: |
| os.mkdir(in_dir) |
| except: |
| pass |
| |
| for i in range(_spread): |
| # files |
| fn = j(in_dir, next_unique_basename('file')) |
| f = open(fn, 'w') |
| f.write('This is %s\n' % fn) |
| f.close() |
| |
| # dirs |
| if (_levels > 1): |
| dn = j(in_dir, next_unique_basename('dir')) |
| create_tree(dn, _levels - 1, _spread) |
| |
| def svn(*args): |
| name = args[0] |
| |
| cmd = [ svn_bin ] |
| cmd.extend( list(args) ) |
| if verbose: |
| print 'svn cmd:', ' '.join(cmd) |
| |
| stdin = None |
| if stdin: |
| stdin_arg = subprocess.PIPE |
| else: |
| stdin_arg = None |
| |
| run.tic(name) |
| try: |
| p = subprocess.Popen(cmd, |
| stdin=stdin_arg, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| shell=False) |
| stdout,stderr = p.communicate(input=stdin) |
| except OSError: |
| stdout = stderr = None |
| finally: |
| run.toc() |
| |
| if verbose: |
| if (stdout): |
| print "STDOUT: [[[\n%s]]]" % ''.join(stdout) |
| if (stderr): |
| print "STDERR: [[[\n%s]]]" % ''.join(stderr) |
| |
| return stdout,stderr |
| |
| |
| def add(*args): |
| return svn('add', *args) |
| |
| def ci(*args): |
| return svn('commit', '-mm', *args) |
| |
| def up(*args): |
| return svn('update', *args) |
| |
| def st(*args): |
| return svn('status', *args) |
| |
| def info(*args): |
| return svn('info', *args) |
| |
| _chars = [chr(x) for x in range(ord('a'), ord('z') +1)] |
| |
| def randstr(len=8): |
| return ''.join( [random.choice(_chars) for i in range(len)] ) |
| |
| def _copy(path): |
| dest = next_unique_basename(path + '_copied') |
| svn('copy', path, dest) |
| |
| def _move(path): |
| dest = path + '_moved' |
| svn('move', path, dest) |
| |
| def _propmod(path): |
| so, se = svn('proplist', path) |
| propnames = [line.strip() for line in so.strip().split('\n')[1:]] |
| |
| # modify? |
| if len(propnames): |
| svn('ps', propnames[len(propnames) / 2], randstr(), path) |
| |
| # del? |
| if len(propnames) > 1: |
| svn('propdel', propnames[len(propnames) / 2], path) |
| |
| def _propadd(path): |
| # set a new one. |
| svn('propset', randstr(), randstr(), path) |
| |
| def _mod(path): |
| if os.path.isdir(path): |
| _propmod(path) |
| return |
| |
| f = open(path, 'a') |
| f.write('\n%s\n' % randstr()) |
| f.close() |
| |
| def _add(path): |
| if os.path.isfile(path): |
| return _mod(path) |
| |
| if random.choice((True, False)): |
| # create a dir |
| svn('mkdir', j(path, next_unique_basename('new_dir'))) |
| else: |
| # create a file |
| new_path = j(path, next_unique_basename('new_file')) |
| f = open(new_path, 'w') |
| f.write(randstr()) |
| f.close() |
| svn('add', new_path) |
| |
| def _del(path): |
| svn('delete', path) |
| |
| _mod_funcs = (_mod, _add, _propmod, _propadd, )#_copy,) # _move, _del) |
| |
| def modify_tree(in_dir, fraction): |
| child_names = os.listdir(in_dir) |
| for child_name in child_names: |
| if child_name[0] == '.': |
| continue |
| if random.random() < fraction: |
| path = j(in_dir, child_name) |
| random.choice(_mod_funcs)(path) |
| |
| for child_name in child_names: |
| if child_name[0] == '.': continue |
| path = j(in_dir, child_name) |
| if os.path.isdir(path): |
| modify_tree(path, fraction) |
| |
| def propadd_tree(in_dir, fraction): |
| for child_name in os.listdir(in_dir): |
| if child_name[0] == '.': continue |
| path = j(in_dir, child_name) |
| if random.random() < fraction: |
| _propadd(path) |
| if os.path.isdir(path): |
| propadd_tree(path, fraction) |
| |
| |
| def rmtree_onerror(func, path, exc_info): |
| """Error handler for ``shutil.rmtree``. |
| |
| If the error is due to an access error (read only file) |
| it attempts to add write permission and then retries. |
| |
| If the error is for another reason it re-raises the error. |
| |
| Usage : ``shutil.rmtree(path, onerror=onerror)`` |
| """ |
| if not os.access(path, os.W_OK): |
| # Is the error an access error ? |
| os.chmod(path, stat.S_IWUSR) |
| func(path) |
| else: |
| raise |
| |
| base = tempfile.mkdtemp() |
| |
| # ensure identical modifications for every run |
| random.seed(0) |
| |
| aborted = True |
| |
| try: |
| repos = j(base, 'repos') |
| repos = repos.replace('\\', '/') |
| wc = j(base, 'wc') |
| wc2 = j(base, 'wc2') |
| |
| if repos.startswith('/'): |
| file_url = 'file://%s' % repos |
| else: |
| file_url = 'file:///%s' % repos |
| |
| print '\nRunning svn benchmark in', base |
| print 'dir levels: %s; new files and dirs per leaf: %s' %( |
| run_kind.levels, run_kind.spread) |
| |
| started = datetime.datetime.now() |
| |
| try: |
| run_cmd([svnadmin_bin, 'create', repos]) |
| svn('checkout', file_url, wc) |
| |
| trunk = j(wc, 'trunk') |
| create_tree(trunk, run_kind.levels, run_kind.spread) |
| add(trunk) |
| st(wc) |
| ci(wc) |
| up(wc) |
| propadd_tree(trunk, 0.05) |
| ci(wc) |
| up(wc) |
| st(wc) |
| info('-R', wc) |
| |
| trunk_url = file_url + '/trunk' |
| branch_url = file_url + '/branch' |
| |
| svn('copy', '-mm', trunk_url, branch_url) |
| st(wc) |
| |
| up(wc) |
| st(wc) |
| info('-R', wc) |
| |
| svn('checkout', trunk_url, wc2) |
| st(wc2) |
| modify_tree(wc2, 0.5) |
| st(wc2) |
| ci(wc2) |
| up(wc2) |
| up(wc) |
| |
| svn('switch', branch_url, wc2) |
| modify_tree(wc2, 0.5) |
| st(wc2) |
| info('-R', wc2) |
| ci(wc2) |
| up(wc2) |
| up(wc) |
| |
| modify_tree(trunk, 0.5) |
| st(wc) |
| ci(wc) |
| up(wc2) |
| up(wc) |
| |
| svn('merge', '--accept=postpone', trunk_url, wc2) |
| st(wc2) |
| info('-R', wc2) |
| svn('resolve', '--accept=mine-conflict', wc2) |
| st(wc2) |
| svn('resolved', '-R', wc2) |
| st(wc2) |
| info('-R', wc2) |
| ci(wc2) |
| up(wc2) |
| up(wc) |
| |
| svn('merge', '--accept=postpone', '--reintegrate', branch_url, trunk) |
| st(wc) |
| svn('resolve', '--accept=mine-conflict', wc) |
| st(wc) |
| svn('resolved', '-R', wc) |
| st(wc) |
| ci(wc) |
| up(wc2) |
| up(wc) |
| |
| svn('delete', j(wc, 'branch')) |
| ci(wc) |
| up(wc) |
| |
| aborted = False |
| |
| finally: |
| stopped = datetime.datetime.now() |
| print '\nDone with svn benchmark in', (stopped - started) |
| |
| run.remember_timing(TOTAL_RUN, |
| timedelta_to_seconds(stopped - started)) |
| finally: |
| run.done(aborted) |
| run.submit_timings() |
| shutil.rmtree(base, onerror=rmtree_onerror) |
| |
| return aborted |
| |
| |
| # --------------------------------------------------------------------- |
| |
| |
| def cmdline_run(db, options, run_kind_str, N=1): |
| run_kind = parse_one_timing_selection(db, run_kind_str) |
| |
| N = int(N) |
| |
| print 'Hi, going to run a Subversion benchmark series of %d runs...' % N |
| print 'Label is %s' % run_kind.label() |
| |
| # can we run the svn binaries? |
| svn_bin = j(options.svn_bin_dir, 'svn') |
| svnadmin_bin = j(options.svn_bin_dir, 'svnadmin') |
| |
| for b in (svn_bin, svnadmin_bin): |
| so,se = run_cmd([b, '--version']) |
| if not so: |
| bail("Can't run %s" % b) |
| |
| print ', '.join([s.strip() for s in so.split('\n')[:2]]) |
| |
| batch = Batch(db) |
| |
| for i in range(N): |
| print 'Run %d of %d' % (i + 1, N) |
| perform_run(batch, run_kind, |
| svn_bin, svnadmin_bin, options.verbose) |
| |
| batch.done() |
| |
| |
| def cmdline_list(db, options, *args): |
| run_kinds = parse_timings_selections(db, *args) |
| |
| for run_kind in run_kinds: |
| |
| constraints = [] |
| def add_if_not_none(name, val): |
| if val: |
| constraints.append(' %s = %s' % (name, val)) |
| add_if_not_none('branch', run_kind.branch) |
| add_if_not_none('revision', run_kind.revision) |
| add_if_not_none('levels', run_kind.levels) |
| add_if_not_none('spread', run_kind.spread) |
| if constraints: |
| print 'For\n', '\n'.join(constraints) |
| print 'I found:' |
| |
| d = TimingQuery(db, run_kind) |
| |
| cmd_names = d.get_sorted_command_names() |
| if cmd_names: |
| print '\n%d command names:\n ' % len(cmd_names), '\n '.join(cmd_names) |
| |
| branches = d.get_sorted_branches() |
| if branches and (len(branches) > 1 or branches[0] != run_kind.branch): |
| print '\n%d branches:\n ' % len(branches), '\n '.join(branches) |
| |
| revisions = d.get_sorted_revisions() |
| if revisions and (len(revisions) > 1 or revisions[0] != run_kind.revision): |
| print '\n%d revisions:\n ' % len(revisions), '\n '.join(revisions) |
| |
| levels_spread = d.get_sorted_levels_spread() |
| if levels_spread and ( |
| len(levels_spread) > 1 |
| or levels_spread[0] != (run_kind.levels, run_kind.spread)): |
| print '\n%d kinds of levels x spread:\n ' % len(levels_spread), '\n '.join( |
| [ ('%dx%d' % (l, s)) for l,s in levels_spread ]) |
| |
| print "\n%d runs in %d batches.\n" % (d.count_runs_batches()) |
| |
| |
| def cmdline_show(db, options, *run_kind_strings): |
| run_kinds = parse_timings_selections(db, *run_kind_strings) |
| for run_kind in run_kinds: |
| q = TimingQuery(db, run_kind) |
| timings = q.get_timings() |
| |
| s = [] |
| s.append('Timings for %s' % run_kind.label()) |
| s.append(' N min max avg operation (unit is seconds)') |
| |
| for command_name in q.get_sorted_command_names(): |
| if options.command_names and command_name not in options.command_names: |
| continue |
| n, tmin, tmax, tavg = timings[command_name] |
| |
| s.append('%4s %7.2f %7.2f %7.2f %s' % ( |
| n_label(n), |
| tmin, |
| tmax, |
| tavg, |
| command_name)) |
| |
| print '\n'.join(s) |
| |
| |
| def cmdline_compare(db, options, *args): |
| run_kinds = parse_timings_selections(db, *args) |
| if len(run_kinds) < 2: |
| bail("Need at least two sets of timings to compare.") |
| |
| |
| left_kind = run_kinds[0] |
| leftq = TimingQuery(db, left_kind) |
| left = leftq.get_timings() |
| if not left: |
| bail("No timings for %s" % left_kind.label()) |
| |
| for run_kind_idx in range(1, len(run_kinds)): |
| right_kind = run_kinds[run_kind_idx] |
| |
| rightq = TimingQuery(db, right_kind) |
| right = rightq.get_timings() |
| if not right: |
| print "No timings for %s" % right_kind.label() |
| continue |
| |
| label = 'Compare %s to %s' % (right_kind.label(), left_kind.label()) |
| |
| s = [label] |
| |
| verbose = options.verbose |
| if not verbose: |
| s.append(' N avg operation') |
| else: |
| s.append(' N min max avg operation') |
| |
| command_names = [name for name in leftq.get_sorted_command_names() |
| if name in right] |
| if options.command_names: |
| command_names = [name for name in command_names |
| if name in options.command_names] |
| |
| for command_name in command_names: |
| left_N, left_min, left_max, left_avg = left[command_name] |
| right_N, right_min, right_max, right_avg = right[command_name] |
| |
| N_str = '%s/%s' % (n_label(left_N), n_label(right_N)) |
| avg_str = '%7.2f|%+7.3f' % (do_div(left_avg, right_avg), |
| do_diff(left_avg, right_avg)) |
| |
| if not verbose: |
| s.append('%9s %-16s %s' % (N_str, avg_str, command_name)) |
| else: |
| min_str = '%7.2f|%+7.3f' % (do_div(left_min, right_min), |
| do_diff(left_min, right_min)) |
| max_str = '%7.2f|%+7.3f' % (do_div(left_max, right_max), |
| do_diff(left_max, right_max)) |
| |
| s.append('%9s %-16s %-16s %-16s %s' % (N_str, min_str, max_str, avg_str, |
| command_name)) |
| |
| s.extend([ |
| '(legend: "1.23|+0.45" means: slower by factor 1.23 and by 0.45 seconds;', |
| ' factor < 1 and seconds < 0 means \'%s\' is faster.' |
| % right_kind.label(), |
| ' "2/3" means: \'%s\' has 2 timings on record, the other has 3.)' |
| % left_kind.label() |
| ]) |
| |
| |
| print '\n'.join(s) |
| |
| |
| # ------------------------------------------------------- charts |
| |
| def cmdline_chart_compare(db, options, *args): |
| import matplotlib |
| matplotlib.use('Agg') |
| import numpy as np |
| import matplotlib.pylab as plt |
| |
| labels = [] |
| timing_sets = [] |
| command_names = None |
| |
| run_kinds = parse_timings_selections(db, *args) |
| |
| # iterate the timings selections and accumulate data |
| for run_kind in run_kinds: |
| query = TimingQuery(db, run_kind) |
| timings = query.get_timings() |
| if not timings: |
| print "No timings for %s" % run_kind.label() |
| continue |
| labels.append(run_kind.label()) |
| timing_sets.append(timings) |
| |
| # it only makes sense to compare those commands that have timings |
| # in the first selection, because that is the one everything else |
| # is compared to. Remember the first selection's command names. |
| if not command_names: |
| command_names = query.get_sorted_command_names() |
| |
| |
| if len(timing_sets) < 2: |
| bail("Not enough timings") |
| |
| if options.command_names: |
| command_names = [name for name in command_names |
| if name in options.command_names] |
| |
| chart_path = options.chart_path |
| if not chart_path: |
| chart_path = 'compare_' + '_'.join( |
| [ filesystem_safe_string(l) for l in labels ] |
| ) + '.svg' |
| |
| N = len(command_names) |
| M = len(timing_sets) - 1 |
| if M < 2: |
| M = 2 |
| |
| group_positions = np.arange(N) # the y locations for the groups |
| dist = 1. / (1. + M) |
| height = (1. - dist) / M # the height of the bars |
| |
| fig = plt.figure(figsize=(12, 5 + 0.2*N*M)) |
| plot1 = fig.add_subplot(121) |
| plot2 = fig.add_subplot(122) |
| |
| left = timing_sets[0] |
| |
| # Iterate timing sets. Each loop produces one bar for each command name |
| # group. |
| for label_i,label in enumerate(labels[1:],1): |
| right = timing_sets[label_i] |
| if not right: |
| continue |
| |
| for cmd_i, command_name in enumerate(command_names): |
| if command_name not in right: |
| #skip |
| continue |
| |
| left_N, left_min, left_max, left_avg = left[command_name] |
| right_N, right_min, right_max, right_avg = right[command_name] |
| |
| div_avg = 100. * (do_div(left_avg, right_avg) - 1.0) |
| if div_avg <= 0: |
| col = '#55dd55' |
| else: |
| col = '#dd5555' |
| |
| diff_val = do_diff(left_avg, right_avg) |
| |
| ofs = (dist + height) / 2. + height * (label_i - 1) |
| |
| barheight = height * (1.0 - dist) |
| |
| y = float(cmd_i) + ofs |
| |
| plot1.barh((y, ), |
| (div_avg, ), |
| barheight, |
| color=col, edgecolor='white') |
| plot1.text(0., y + height/2., |
| '%s %+5.1f%%' % (label, div_avg), |
| ha='right', va='center', size='small', |
| rotation=0, family='monospace') |
| |
| plot2.barh((y, ), |
| (diff_val, ), |
| barheight, |
| color=col, edgecolor='white') |
| plot2.text(0., y + height/2., |
| '%s %+6.2fs' % (label, diff_val), |
| ha='right', va='center', size='small', |
| rotation=0, family='monospace') |
| |
| |
| for p in (plot1, plot2): |
| xlim = list(p.get_xlim()) |
| if xlim[1] < 10.: |
| xlim[1] = 10. |
| # make sure the zero line is far enough right so that the annotations |
| # fit inside the chart. About half the width should suffice. |
| if xlim[0] > -xlim[1]: |
| xlim[0] = -xlim[1] |
| p.set_xlim(*xlim) |
| p.set_xticks((0,)) |
| p.set_yticks(group_positions + (height / 2.)) |
| p.set_yticklabels(()) |
| p.set_ylim((len(command_names), 0)) |
| p.grid() |
| |
| plot1.set_xticklabels(('+-0%',), rotation=0) |
| plot1.set_title('Average runtime change from %s in %%' % labels[0], |
| size='medium') |
| |
| plot2.set_xticklabels(('+-0s',), rotation=0) |
| plot2.set_title('Average runtime change from %s in seconds' % labels[0], |
| size='medium') |
| |
| margin = 1./(2 + N*M) |
| titlemargin = 0 |
| if options.title: |
| titlemargin = margin * 1.5 |
| |
| fig.subplots_adjust(left=0.005, right=0.995, wspace=0.3, bottom=margin, |
| top=1.0-margin-titlemargin) |
| |
| ystep = (1.0 - 2.*margin - titlemargin) / len(command_names) |
| |
| for idx,command_name in enumerate(command_names): |
| ylabel = '%s\nvs. %.1fs' % ( |
| command_name, |
| left[command_name][3]) |
| |
| ypos=1.0 - margin - titlemargin - ystep/M - ystep * idx |
| plt.figtext(0.5, ypos, |
| command_name, |
| ha='center', va='top', |
| size='medium', weight='bold') |
| plt.figtext(0.5, ypos - ystep/(M+1), |
| '%s\n= %.2fs' % ( |
| labels[0], left[command_name][3]), |
| ha='center', va='top', |
| size='small') |
| |
| if options.title: |
| plt.figtext(0.5, 1. - titlemargin/2, options.title, ha='center', |
| va='center', weight='bold') |
| |
| plt.savefig(chart_path) |
| print 'wrote chart file:', chart_path |
| |
| |
| # ------------------------------------------------------------ main |
| |
| |
| # Custom option formatter, keeping newlines in the description. |
| # adapted from: |
| # http://groups.google.com/group/comp.lang.python/msg/09f28e26af0699b1 |
| import textwrap |
| class IndentedHelpFormatterWithNL(optparse.IndentedHelpFormatter): |
| def format_description(self, description): |
| if not description: return "" |
| desc_width = self.width - self.current_indent |
| indent = " "*self.current_indent |
| bits = description.split('\n') |
| formatted_bits = [ |
| textwrap.fill(bit, |
| desc_width, |
| initial_indent=indent, |
| subsequent_indent=indent) |
| for bit in bits] |
| result = "\n".join(formatted_bits) + "\n" |
| return result |
| |
| if __name__ == '__main__': |
| parser = optparse.OptionParser(formatter=IndentedHelpFormatterWithNL()) |
| # -h is automatically added. |
| ### should probably expand the help for that. and see about -? |
| parser.add_option('-v', '--verbose', action='store_true', dest='verbose', |
| help='Verbose operation') |
| parser.add_option('-b', '--svn-bin-dir', action='store', dest='svn_bin_dir', |
| default='', |
| help='Specify directory to find Subversion binaries in') |
| parser.add_option('-f', '--db-path', action='store', dest='db_path', |
| default='benchmark.db', |
| help='Specify path to SQLite database file') |
| parser.add_option('-o', '--chart-path', action='store', dest='chart_path', |
| help='Supply a path for chart output.') |
| parser.add_option('-c', '--command-names', action='store', |
| dest='command_names', |
| help='Comma separated list of command names to limit to.') |
| parser.add_option('-t', '--title', action='store', |
| dest='title', |
| help='For charts, a title to print in the chart graphics.') |
| |
| parser.set_description(__doc__) |
| parser.set_usage('') |
| |
| |
| options, args = parser.parse_args() |
| |
| def usage(msg=None): |
| parser.print_help() |
| if msg: |
| print |
| print msg |
| bail() |
| |
| # there should be at least one arg left: the sub-command |
| if not args: |
| usage('No command argument supplied.') |
| |
| cmd = args[0] |
| del args[0] |
| |
| db = TimingsDb(options.db_path) |
| |
| if cmd == 'run': |
| if len(args) < 1 or len(args) > 2: |
| usage() |
| cmdline_run(db, options, *args) |
| |
| elif cmd == 'compare': |
| if len(args) < 2: |
| usage() |
| cmdline_compare(db, options, *args) |
| |
| elif cmd == 'list': |
| cmdline_list(db, options, *args) |
| |
| elif cmd == 'show': |
| cmdline_show(db, options, *args) |
| |
| elif cmd == 'chart': |
| if 'compare'.startswith(args[0]): |
| cmdline_chart_compare(db, options, *args[1:]) |
| else: |
| usage() |
| |
| else: |
| usage('Unknown subcommand argument: %s' % cmd) |
