share/test/run-perf.sh - avro - Git at Google

 #!/bin/bash

 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
 #
 #     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 set -ex

 function usage {
   echo "`basename $0` --help"
   echo "`basename $0` [--min] [--out-dir D] [--iters N] [--skip-one]\\"
   echo "    [--only-combine] [--perf-args STRING] [-Dkey=value]* \\"
   echo "    [--] [-Dkey=value]* branch_1[:name_1] .. [-Dkey=value]* branch_n[:name_n]"
   echo
   echo "Run a set of trials Perf.java trials can compare results."
   echo "A 'trial' is N runs of Perf.java against the code as it"
   echo "exists on a branch in git.  By comparing Perf.java output"
   echo "generated by different branches in git, we can understand"
   echo "the relative performance of those branches."
   echo
   echo "This script must be run in the lang/java/ipc directory of"
   echo "the Avro source code, on a computer where Maven is installed"
   echo "and the other build-prerequisites of Avro are in place.  This"
   echo "script will do a 'mvn clean install' of Avro from withing"
   echo "the lang/java directory, before running tests."
   echo
   echo "The way Perf.java works is that it times an 'inner loop' that"
   echo "reads or writes a large number of records (the exact number can"
   echo "be controlled by a system property as described below).  This"
   echo "inner loop is called a 'cycle.'  Perf.java runs a medium number of"
   echo "these cycles, and outputs either the average or the minimum"
   echo "of their running times.  This script runs Perf.java a small number"
   echo "of times (controllable by the --iters flag), and takes either the"
   echo "average or minimum of those.  The result of all this is the results"
   echo "of a single 'trial.'"
   echo
   echo "The basic model is that there is a 'baseline' trial plus any"
   echo "number of 'treatment' trials.  The goal is to compare the"
   echo "performance of each treatment against the baseline.  The main"
   echo "output is written to the file 'summary.csv'.  This file contains"
   echo "one line per performance test run by Perf.java.  Each row contains"
   echo "a 'results' column for each trial, followed by a 'comparison' column"
   echo "for each treatment trial.  The results column contains the average"
   echo "(or minimum) of the runtimes of all cycles over all iterations of"
   echo "the trial.  The comparison columns contains the difference between"
   echo "the performance of the treatment and the baseline, as a percent"
   echo "of the baseline.  Specifically, it countains"
   echo "  100*(treatment-baseline)/baseline, i.e., positive numbers mean"
   echo "we've seen a speedup."
   echo
   echo "By default, the running times of cycles are averaged together."
   echo "The --min flag changes that to taking the minimum."
   echo
   echo "By default, output is written to the current working directory."
   echo "However, lots of intermediate files are generated, so it's recommended"
   echo "that the --out-dir argument is used to redirect the output to"
   echo "a different working directory."
   echo
   echo "By default, the number of iterations in a trial is 4, but this can"
   echo "be changed with the --iters flag."
   echo
   echo "Perf.java takes a number of command-line arguments, and can be"
   echo "influenced by system properties.  Command-line arguments can be"
   echo "passed using the --perf-args flag.  When using this switch, pass"
   echo "your Perf.java command-line arguments in a single string, even if"
   echo "there are more than one of them.  You can set system properties"
   echo " using -Dkey=value switch, just as you would with Maven. System"
   echo "properties that come before the the '--' switch and the first"
   echo "branch are passed to all trials. System properties that come after"
   echo "the '--' switch and/or first branch are passed to the branch that"
   echo "follows them.  Commonly used system properties include:"
   echo
   echo "     org.apache.avro.io.perf.count -- the number of elements"
   echo "generated for the inner-most loop of the performance test.  Defaults"
   echo "to 250K.  Must be a multple of 4."
   echo
   echo "     org.apache.avro.io.perf.cycles -- the number of times the inner-"
   echo "most loop is called within an invocation of Perf.java.  Defaults"
   echo " to 800."
   echo
   echo "     org.apache.avro.io.perf.use-direct -- use DirectBinaryEncoder instead"
   echo "of BufferedBinaryEncoder for write tests.  It is slower, but performance-wise"
   echo "it can be more consistent, which helps when trying to detect small performance"
   echo "improvements."
   echo
   echo "     org.apache.avro.specific.use_custom_coders -- flag that turns on"
   echo " the use of the custom-coder optimization in the SpecificRecord tests."
   echo "Defaults to 'false;' set to 'true' to turn them on."
   echo
   echo "Trials, as indicated, are branches in git.  The branch_i arguments"
   echo " indicate which what branches make up a trial.  The first of these"
   echo "(branch_1) is considered the \"baseline\" trial: it's the trial"
   echo "that all the others are compared against.  (However, if the --skip-one"
   echo "is provided, the result from the first trial is ignored and the second"
   echo "becomes the baseline.)"
   echo
   echo "Each trial has a name as well as a branch.  By default, the name of"
   echo "the branch is the name of the trial, but an explicit name can be given"
   echo "by suffixing the branch name with a trial name (e.g., 'foo:bar' will"
   echo "use the branch 'foo' for a trial, but the trial will be named 'bar')."
   echo "Trials must have unique names, so when multiple trials are run off the"
   echo "same branch, explicit trial names must be used."
   echo
   echo "In addition to writing 'summary.csv', this script outputs other files,"
   echo "allowing you to analyze the granular results of a test run.  The file"
   echo "results.csv contains a row per test in Perf.java.  Each column"
   echo "contains the result of a single run of Perf.java.  If N is the"
   echo "number of iterations in a trial, then the first N columns are the"
   echo "results from the individual iterations of the first trial, the"
   echo "next N are the results from the second trial, and so forth.  In"
   echo "addition, for each branch B being tested, there are multiple"
   echo "files 'B_i.csv' for each iteration i in the trial.  These per-trial"
   echo "files have two columns, the first being the name of the test, the"
   echo "second being the result of that test.  Thus, 'result.csv' is the"
   echo "result of joining these per-trial files on the trial-name, and"
   echo "summary.csv averages (or takes the minimum) of these per-trial"
   echo "results, and adds the comparison column."
   echo
   echo "If the --only-combine flag is given, then the script will assume"
   echo "that the B_i files have been generated, and will simply join them"
   echo "to compute results.csv and summary.csv.  This allows you to debug"
   echo "the code that combines these files without having to wait around"
   echo "for Perf.java to be run a bunch of times."
 }

 if [[ "$1" == "--help" ]]; then
   usage
   exit 0
 fi

 if [[ ! `pwd` =~ java/ipc ]]; then
   echo "Must be run from lang/java/ipc"
   echo "Type `basename $0` --help for help"
   exit 1
 fi

 TEST="-c nt"
 EXTRA_CLI=""
 OUT="."
 SKIP_ONE="false"
 STATIC_SYSPROPS=()
 ITERS=4

 # DBG=echo

 function Perf_java {
   local fname=$1
   shift

   if [[ "$DBG" != "" ]]; then
     $DBG MAVEN_OPTS=-server mvn exec:java -Dexec.classpathScope=test \
       -Dexec.mainClass=org.apache.avro.io.Perf ${STATIC_SYSPROPS[@]} \
       -Dexec.args="${TEST} -o ${fname} ${EXTRA_CLI}" \
       $@
   else
     mvn exec:java -Dexec.classpathScope=test \
       -Dexec.mainClass=org.apache.avro.io.Perf ${STATIC_SYSPROPS[@]} \
       -Dexec.args="${TEST} -o ${fname} ${EXTRA_CLI}" \
       $@
   fi
 }

 function run_trial {
   local lastbranch=$1
   local thisbranch=$2
   local thistrialname=$3
   shift 3

   if [[ "$thisbranch" != "$lastbranch" ]]; then
     $DBG git checkout $thisbranch
     (cd ..; $DBG mvn clean && $DBG mvn -pl "avro,compiler,maven-plugin,ipc" install -DskipTests)
   fi
   for i in $(seq 1 ${ITERS}); do Perf_java ${OUT}/${thistrialname}${i}.csv $@; done
 }

 function run_trials {
   local -a allprops=( )

   while (( "$#" )); do
     case "$1" in
       --)
         break;
         ;;
       *)
         allprops+=( $1 )
         shift
         ;;
     esac
   done

   local -a thisprops=( )
   local lastbranch=""
   local thisbranch
   local thistrialname

   while (( "$#" )); do
     case "$1" in
       --) # Ignore these
         shift
         ;;
       -D*)
         thisprops+=( $1 )
         shift
         ;;
       *)
         thisbranch=$1
         thistrialname=$2
         git rev-parse --verify $thisbranch
         run_trial "$lastbranch" $thisbranch $thistrialname ${allprops[@]} ${thisprops[@]}
         lastbranch=$thisbranch
         thisprops=( )
         shift 2
         ;;
     esac
   done
 }

 function join_results {
   pushd ${OUT}
   local header="TestName"
   for b in $@; do
     for i in $(seq 1 ${ITERS}); do
       header="${header},${b}${i}"
     done
   done
 #  echo $header > results.csv
   if [[ "$SKIP_ONE" == "true" ]]; then shift; fi
   cut -d , -f 1,2 ${1}1.csv | sort >> results.csv
   if [[ 1 < "${ITERS}" ]]; then
     for i in $(seq 2 ${ITERS}); do
       cut -d , -f 1,2 ${1}$i.csv | sort | join -t , results.csv - > tmp.csv
       mv tmp.csv results.csv
     done
   fi
   shift
   for b in $@; do
     for i in $(seq 1 ${ITERS}); do
       cut -d , -f 1,2 ${b}$i.csv | sort | join -t , results.csv - > tmp.csv
       mv tmp.csv results.csv
     done
   done
   popd
 }

 AVG='BEGIN { RS=" "; } { s += $1; n += 1; } END { printf "%f", s/n; }'
 MIN='BEGIN { RS=" "; m = 10000000000; } { if ($1 < m) m = $1; } END { printf "%f", m; }'
 PERCENT='{ printf "%f", 100*($1-$2)/$1; }'

 function print_line {
   local line=$1
   shift
   local awks
   if [[ "$TEST" == "-c nt" ]]; then awks="$AVG"; else awks="$MIN"; fi

   local -a results=( )
   for t in ${trials[*]}; do
     local result=""
     for i in $(seq 1 $ITERS); do
       result="$result $1"
       shift
     done
     result=$(echo $result | awk "$awks")
     results+=( $result )
     line="${line},${result}"
   done

   local baseline=0
   if [[ "$SKIP_ONE" == "true" ]]; then start=1; fi
   for i in $(seq `expr ${baseline} + 1` `expr ${#trials[*]} - 1`); do
     result=$(echo "${results[$baseline]} ${results[$i]}" | awk "$PERCENT")
     line="${line},${result}"
   done
   echo "$line"
 }


 ###
 ### ACTUAL SCRIPT STARTS HERE
 ###

 declare command="$0 $*"
 declare onlycombine="false"
 declare -a run_trials_args=( )
 declare -a trials=( )

 while (( "$#" )); do
   case "$1" in
     --help)
       usage
       exit
       ;;
     --min)
       TEST="-c nm"
       shift
       ;;
     --out-dir)
       if [[ $OUT != "." ]]; then
         echo "Cannot use --out-dir twice."
         echo "Type `basename $0` --help for help"
         exit 1
       fi
       OUT=$2
       mkdir -p $OUT
       shift 2
       ;;
     --iters)
       ITERS=$2
       shift 2
       ;;
     --only-combine)
       onlycombine="true"
       shift
       ;;
     --skip-one)
       SKIP_ONE="true"
       shift
       ;;
     --perf-args)
       EXTRA_CLI=$2
       shift 2
       ;;
     -D*)
       if [[ ! $1 =~ ^-D[^\ =]+= ]]; then
         echo "Bad system property: $1"
         echo "Type `basename $0` --help for help"
         exit 1
       fi
       run_trials_args+=( $1 )
       shift
       ;;
     --)
       run_trials_args+=( $1 )
       shift
       ;;
     --*)
       echo "Unknown switch: $1"
       echo "Type `basename $0` --help for help"
       exit 1
       ;;
     *)
       if [[ "$1" =~ ^([^:]*):(.*) ]]; then
         thisbranch=${BASH_REMATCH[1]}
         thistrialname=${BASH_REMATCH[2]}
       else
         thisbranch=$1
         thistrialname=$1
       fi
       if [[ "$thisbranch" == "" || "$thistrialname" == "" ]]; then
         echo "Neither branch ($thisbranch) nor trial ($thistrialname) names may be empty"
         echo "Type `basename $0` --help for help"
         exit 1
       fi
       if [[ "${trials[@]}" =~ $thistrialname ]]; then
         echo "Trial named '$thistrialname' is not unique"
         echo "Type `basename $0` --help for help"
         exit 1
       fi
       trials+=( "$thistrialname" )
       run_trials_args+=( "--" "$thisbranch" "$thistrialname" )
       shift
       ;;
   esac
 done

 # Document how the outputs were generated
 echo "$command" > $OUT/command.txt

 if [[ ${onlycombine} == "false" ]]; then
   run_trials ${run_trials_args[@]}
 fi

 join_results ${trials[@]}

 cat $OUT/results.csv | while read line; do
   fields=( $(echo $line | tr "," " ") )
   print_line "${fields[@]}"
 done > $OUT/summary.csv
	#!/bin/bash

	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# https://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	set -ex

	function usage {
	echo "`basename $0` --help"
	echo "`basename $0` [--min] [--out-dir D] [--iters N] [--skip-one]\\"
	echo " [--only-combine] [--perf-args STRING] [-Dkey=value]* \\"
	echo " [--] [-Dkey=value]* branch_1[:name_1] .. [-Dkey=value]* branch_n[:name_n]"
	echo
	echo "Run a set of trials Perf.java trials can compare results."
	echo "A 'trial' is N runs of Perf.java against the code as it"
	echo "exists on a branch in git. By comparing Perf.java output"
	echo "generated by different branches in git, we can understand"
	echo "the relative performance of those branches."
	echo
	echo "This script must be run in the lang/java/ipc directory of"
	echo "the Avro source code, on a computer where Maven is installed"
	echo "and the other build-prerequisites of Avro are in place. This"
	echo "script will do a 'mvn clean install' of Avro from withing"
	echo "the lang/java directory, before running tests."
	echo
	echo "The way Perf.java works is that it times an 'inner loop' that"
	echo "reads or writes a large number of records (the exact number can"
	echo "be controlled by a system property as described below). This"
	echo "inner loop is called a 'cycle.' Perf.java runs a medium number of"
	echo "these cycles, and outputs either the average or the minimum"
	echo "of their running times. This script runs Perf.java a small number"
	echo "of times (controllable by the --iters flag), and takes either the"
	echo "average or minimum of those. The result of all this is the results"
	echo "of a single 'trial.'"
	echo
	echo "The basic model is that there is a 'baseline' trial plus any"
	echo "number of 'treatment' trials. The goal is to compare the"
	echo "performance of each treatment against the baseline. The main"
	echo "output is written to the file 'summary.csv'. This file contains"
	echo "one line per performance test run by Perf.java. Each row contains"
	echo "a 'results' column for each trial, followed by a 'comparison' column"
	echo "for each treatment trial. The results column contains the average"
	echo "(or minimum) of the runtimes of all cycles over all iterations of"
	echo "the trial. The comparison columns contains the difference between"
	echo "the performance of the treatment and the baseline, as a percent"
	echo "of the baseline. Specifically, it countains"
	echo " 100*(treatment-baseline)/baseline, i.e., positive numbers mean"
	echo "we've seen a speedup."
	echo
	echo "By default, the running times of cycles are averaged together."
	echo "The --min flag changes that to taking the minimum."
	echo
	echo "By default, output is written to the current working directory."
	echo "However, lots of intermediate files are generated, so it's recommended"
	echo "that the --out-dir argument is used to redirect the output to"
	echo "a different working directory."
	echo
	echo "By default, the number of iterations in a trial is 4, but this can"
	echo "be changed with the --iters flag."
	echo
	echo "Perf.java takes a number of command-line arguments, and can be"
	echo "influenced by system properties. Command-line arguments can be"
	echo "passed using the --perf-args flag. When using this switch, pass"
	echo "your Perf.java command-line arguments in a single string, even if"
	echo "there are more than one of them. You can set system properties"
	echo " using -Dkey=value switch, just as you would with Maven. System"
	echo "properties that come before the the '--' switch and the first"
	echo "branch are passed to all trials. System properties that come after"
	echo "the '--' switch and/or first branch are passed to the branch that"
	echo "follows them. Commonly used system properties include:"
	echo
	echo " org.apache.avro.io.perf.count -- the number of elements"
	echo "generated for the inner-most loop of the performance test. Defaults"
	echo "to 250K. Must be a multple of 4."
	echo
	echo " org.apache.avro.io.perf.cycles -- the number of times the inner-"
	echo "most loop is called within an invocation of Perf.java. Defaults"
	echo " to 800."
	echo
	echo " org.apache.avro.io.perf.use-direct -- use DirectBinaryEncoder instead"
	echo "of BufferedBinaryEncoder for write tests. It is slower, but performance-wise"
	echo "it can be more consistent, which helps when trying to detect small performance"
	echo "improvements."
	echo
	echo " org.apache.avro.specific.use_custom_coders -- flag that turns on"
	echo " the use of the custom-coder optimization in the SpecificRecord tests."
	echo "Defaults to 'false;' set to 'true' to turn them on."
	echo
	echo "Trials, as indicated, are branches in git. The branch_i arguments"
	echo " indicate which what branches make up a trial. The first of these"
	echo "(branch_1) is considered the \"baseline\" trial: it's the trial"
	echo "that all the others are compared against. (However, if the --skip-one"
	echo "is provided, the result from the first trial is ignored and the second"
	echo "becomes the baseline.)"
	echo
	echo "Each trial has a name as well as a branch. By default, the name of"
	echo "the branch is the name of the trial, but an explicit name can be given"
	echo "by suffixing the branch name with a trial name (e.g., 'foo:bar' will"
	echo "use the branch 'foo' for a trial, but the trial will be named 'bar')."
	echo "Trials must have unique names, so when multiple trials are run off the"
	echo "same branch, explicit trial names must be used."
	echo
	echo "In addition to writing 'summary.csv', this script outputs other files,"
	echo "allowing you to analyze the granular results of a test run. The file"
	echo "results.csv contains a row per test in Perf.java. Each column"
	echo "contains the result of a single run of Perf.java. If N is the"
	echo "number of iterations in a trial, then the first N columns are the"
	echo "results from the individual iterations of the first trial, the"
	echo "next N are the results from the second trial, and so forth. In"
	echo "addition, for each branch B being tested, there are multiple"
	echo "files 'B_i.csv' for each iteration i in the trial. These per-trial"
	echo "files have two columns, the first being the name of the test, the"
	echo "second being the result of that test. Thus, 'result.csv' is the"
	echo "result of joining these per-trial files on the trial-name, and"
	echo "summary.csv averages (or takes the minimum) of these per-trial"
	echo "results, and adds the comparison column."
	echo
	echo "If the --only-combine flag is given, then the script will assume"
	echo "that the B_i files have been generated, and will simply join them"
	echo "to compute results.csv and summary.csv. This allows you to debug"
	echo "the code that combines these files without having to wait around"
	echo "for Perf.java to be run a bunch of times."
	}

	if [[ "$1" == "--help" ]]; then
	usage
	exit 0
	fi

	if [[ ! `pwd` =~ java/ipc ]]; then
	echo "Must be run from lang/java/ipc"
	echo "Type `basename $0` --help for help"
	exit 1
	fi

	TEST="-c nt"
	EXTRA_CLI=""
	OUT="."
	SKIP_ONE="false"
	STATIC_SYSPROPS=()
	ITERS=4

	# DBG=echo

	function Perf_java {
	local fname=$1
	shift

	if [[ "$DBG" != "" ]]; then
	$DBG MAVEN_OPTS=-server mvn exec:java -Dexec.classpathScope=test \
	-Dexec.mainClass=org.apache.avro.io.Perf ${STATIC_SYSPROPS[@]} \
	-Dexec.args="${TEST} -o ${fname} ${EXTRA_CLI}" \
	$@
	else
	mvn exec:java -Dexec.classpathScope=test \
	-Dexec.mainClass=org.apache.avro.io.Perf ${STATIC_SYSPROPS[@]} \
	-Dexec.args="${TEST} -o ${fname} ${EXTRA_CLI}" \
	$@
	fi
	}

	function run_trial {
	local lastbranch=$1
	local thisbranch=$2
	local thistrialname=$3
	shift 3

	if [[ "$thisbranch" != "$lastbranch" ]]; then
	$DBG git checkout $thisbranch
	(cd ..; $DBG mvn clean && $DBG mvn -pl "avro,compiler,maven-plugin,ipc" install -DskipTests)
	fi
	for i in $(seq 1 ${ITERS}); do Perf_java ${OUT}/${thistrialname}${i}.csv $@; done
	}

	function run_trials {
	local -a allprops=( )

	while (( "$#" )); do
	case "$1" in
	--)
	break;
	;;
	*)
	allprops+=( $1 )
	shift
	;;
	esac
	done

	local -a thisprops=( )
	local lastbranch=""
	local thisbranch
	local thistrialname

	while (( "$#" )); do
	case "$1" in
	--) # Ignore these
	shift
	;;
	-D*)
	thisprops+=( $1 )
	shift
	;;
	*)
	thisbranch=$1
	thistrialname=$2
	git rev-parse --verify $thisbranch
	run_trial "$lastbranch" $thisbranch $thistrialname ${allprops[@]} ${thisprops[@]}
	lastbranch=$thisbranch
	thisprops=( )
	shift 2
	;;
	esac
	done
	}

	function join_results {
	pushd ${OUT}
	local header="TestName"
	for b in $@; do
	for i in $(seq 1 ${ITERS}); do
	header="${header},${b}${i}"
	done
	done
	# echo $header > results.csv
	if [[ "$SKIP_ONE" == "true" ]]; then shift; fi
	cut -d , -f 1,2 ${1}1.csv \| sort >> results.csv
	if [[ 1 < "${ITERS}" ]]; then
	for i in $(seq 2 ${ITERS}); do
	cut -d , -f 1,2 ${1}$i.csv \| sort \| join -t , results.csv - > tmp.csv
	mv tmp.csv results.csv
	done
	fi
	shift
	for b in $@; do
	for i in $(seq 1 ${ITERS}); do
	cut -d , -f 1,2 ${b}$i.csv \| sort \| join -t , results.csv - > tmp.csv
	mv tmp.csv results.csv
	done
	done
	popd
	}

	AVG='BEGIN { RS=" "; } { s += $1; n += 1; } END { printf "%f", s/n; }'
	MIN='BEGIN { RS=" "; m = 10000000000; } { if ($1 < m) m = $1; } END { printf "%f", m; }'
	PERCENT='{ printf "%f", 100*($1-$2)/$1; }'

	function print_line {
	local line=$1
	shift
	local awks
	if [[ "$TEST" == "-c nt" ]]; then awks="$AVG"; else awks="$MIN"; fi

	local -a results=( )
	for t in ${trials[*]}; do
	local result=""
	for i in $(seq 1 $ITERS); do
	result="$result $1"
	shift
	done
	result=$(echo $result \| awk "$awks")
	results+=( $result )
	line="${line},${result}"
	done

	local baseline=0
	if [[ "$SKIP_ONE" == "true" ]]; then start=1; fi
	for i in $(seq `expr ${baseline} + 1` `expr ${#trials[*]} - 1`); do
	result=$(echo "${results[$baseline]} ${results[$i]}" \| awk "$PERCENT")
	line="${line},${result}"
	done
	echo "$line"
	}



	###
	### ACTUAL SCRIPT STARTS HERE
	###

	declare command="$0 $*"
	declare onlycombine="false"
	declare -a run_trials_args=( )
	declare -a trials=( )

	while (( "$#" )); do
	case "$1" in
	--help)
	usage
	exit
	;;
	--min)
	TEST="-c nm"
	shift
	;;
	--out-dir)
	if [[ $OUT != "." ]]; then
	echo "Cannot use --out-dir twice."
	echo "Type `basename $0` --help for help"
	exit 1
	fi
	OUT=$2
	mkdir -p $OUT
	shift 2
	;;
	--iters)
	ITERS=$2
	shift 2
	;;
	--only-combine)
	onlycombine="true"
	shift
	;;
	--skip-one)
	SKIP_ONE="true"
	shift
	;;
	--perf-args)
	EXTRA_CLI=$2
	shift 2
	;;
	-D*)
	if [[ ! $1 =~ ^-D[^\ =]+= ]]; then
	echo "Bad system property: $1"
	echo "Type `basename $0` --help for help"
	exit 1
	fi
	run_trials_args+=( $1 )
	shift
	;;
	--)
	run_trials_args+=( $1 )
	shift
	;;
	--*)
	echo "Unknown switch: $1"
	echo "Type `basename $0` --help for help"
	exit 1
	;;
	*)
	if [[ "$1" =~ ^([^:]):(.) ]]; then
	thisbranch=${BASH_REMATCH[1]}
	thistrialname=${BASH_REMATCH[2]}
	else
	thisbranch=$1
	thistrialname=$1
	fi
	if [[ "$thisbranch" == "" \|\| "$thistrialname" == "" ]]; then
	echo "Neither branch ($thisbranch) nor trial ($thistrialname) names may be empty"
	echo "Type `basename $0` --help for help"
	exit 1
	fi
	if [[ "${trials[@]}" =~ $thistrialname ]]; then
	echo "Trial named '$thistrialname' is not unique"
	echo "Type `basename $0` --help for help"
	exit 1
	fi
	trials+=( "$thistrialname" )
	run_trials_args+=( "--" "$thisbranch" "$thistrialname" )
	shift
	;;
	esac
	done

	# Document how the outputs were generated
	echo "$command" > $OUT/command.txt

	if [[ ${onlycombine} == "false" ]]; then
	run_trials ${run_trials_args[@]}
	fi

	join_results ${trials[@]}

	cat $OUT/results.csv \| while read line; do
	fields=( $(echo $line \| tr "," " ") )
	print_line "${fields[@]}"
	done > $OUT/summary.csv