test/perf/src/org/apache/xmlbeans/test/performance/utils/RunComparator.java

/*   Copyright 2004 The Apache Software Foundation
*
*   Licensed under the Apache License, Version 2.0 (the "License");
*   you may not use this file except in compliance with the License.
*   You may obtain a copy of the License at
*
*       http://www.apache.org/licenses/LICENSE-2.0
*
*   Unless required by applicable law or agreed to in writing, software
*   distributed under the License is distributed on an "AS IS" BASIS,
*   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*   See the License for the specific language governing permissions and
*  limitations under the License.
*/
package org.apache.xmlbeans.test.performance.utils;

import org.apache.xmlbeans.XmlError;
import org.apache.xmlbeans.XmlOptions;
import org.openuri.perf.Custom;
import org.openuri.perf.Result;
import org.openuri.perf.ResultSetDocument;

import java.io.File;
import java.io.FileNotFoundException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;


public class RunComparator
{
    // This class compares the xml results file of 2 perf runs and checks differences in the numbers for each test case
    // An optional 'tolerance' % deviation value can be provided which will result in only the cases that are beyond this
    // value (+/-) getting reported

    private ResultSetDocument resultsDocOne;    // XmlObject for the first result set
    private ResultSetDocument resultsDocTwo;    // and the second
    private double tolerance;                   // % deviation in test run numbers

    // The computation of deviation for a given test scase is done as follows:
    // (run1 - run2)/run1 as a %. Negative value implies regression, +ve implies improvement


    public RunComparator(String runResultsXmlFilePath1, String runResultsXmlFilePath2, double tolerance) throws Exception
    {
        System.out.println("XMLBeans RunComparator - comparing Perf Runs...");

        if ((runResultsXmlFilePath1 != null) && (runResultsXmlFilePath2 != null))
        {
            try
            {
                File resultsFileOne = new File(runResultsXmlFilePath1);
                File resultsFileTwo = new File(runResultsXmlFilePath2);

                if (tolerance > 100 || tolerance < 0)
                {
                    throw(new Exception("Invalid input value for 'tolerance'"));
                }
                this.tolerance = tolerance;

                // generate XmlObjects and validate against schema
                XmlOptions validateOptions = new XmlOptions();
                List errors = new ArrayList();
                validateOptions.setErrorListener(errors);

                resultsDocOne = ResultSetDocument.Factory.parse(resultsFileOne);
                if (!resultsDocOne.validate(validateOptions))
                {
                    System.out.println("Result Set XML Document " + runResultsXmlFilePath1 + "is not Valid against schema!");
                    for (Iterator iterator = errors.iterator(); iterator.hasNext();)
                    {
                        XmlError eachErr = (XmlError) iterator.next();
                        //System.out.println("Validation Error:" + eachErr.getMessage() + eachErr.getLine());
                    }
                    //throw new Exception("Invalid Results File " + runResultsXmlFilePath1);
                }

                errors.clear();
                resultsDocTwo = ResultSetDocument.Factory.parse(resultsFileTwo);
                if (!resultsDocTwo.validate(validateOptions))
                {
                    System.out.println("Result Set XML Document " + runResultsXmlFilePath2 + "is not Valid against schema!");
                    for (Iterator iterator = errors.iterator(); iterator.hasNext();)
                    {
                        XmlError eachErr = (XmlError) iterator.next();
                        //System.out.println("Validation Error:" + eachErr.getMessage() + eachErr.getLine());
                    }

                    //throw new Exception("Invalid Results File " + runResultsXmlFilePath2);
                }

            }

            catch (FileNotFoundException fne)
            {
                throw new Exception("Invalid results file(s) specified!");
            }
            catch (Exception e)
            {
                e.printStackTrace();
            }
        }
        else
        {
            throw new Exception("Input File Path(s) in null!");
        }

    } // end constructor


    private void compare()
    {
        // create a hashmap with first run results
        Result[] res1 = resultsDocOne.getResultSet().getResultArray();

        HashMap runOneHashMap = new HashMap();
        for (int i = 0; i < res1.length; i++)
        {
            Result eachResult = res1[i];
            //System.out.println("Name:" + eachResult.getName() + "\tnote:" + eachResult.getNote() + "\tTime:" + eachResult.getTime());

            Custom[] custArr = eachResult.getCustomArray();
            if ((custArr != null) || (custArr.length > 0))
            {
                for (int j=0; j < custArr.length; j++)
                {
                    Custom eachCustomElem = custArr[j];
                    //System.out.println("Cust:" + eachCustomElem.getName() + "\tvalue:" + eachCustomElem.getValue());

                    String hashKey = eachResult.getName() + "_" + eachCustomElem.getName() + "_" + eachCustomElem.getValue();
                    //System.out.println("Key:" + hashKey);
                    //System.out.println("Val:" + eachResult.getTime());

                    // insert each result into the hashmap
                    runOneHashMap.put(hashKey, eachResult);
                }
            }
            else
            {
                // every result should have a custom element but this is not always true, the generated results are invalid sometimes
                System.out.println("Result Set 1, Custom Element Missing!:" + eachResult.getName());
            }
        }

        // now iterate thro second result set and compare
        Result[] res2 = resultsDocTwo.getResultSet().getResultArray();
        for (int i = 0; i < res2.length; i++)
        {
            Result eachResult2 = res2[i];

            Custom[] custArr2 = eachResult2.getCustomArray();
            if ((custArr2 != null) || (custArr2.length > 0))
            {
                for (int j = 0; j < custArr2.length; j++)
                {
                    Custom custom = custArr2[j];

                    String hashKeyToLookFor = eachResult2.getName() + "_" + custom.getName() + "_" + custom.getValue();
                    Result resFromHash = (Result) runOneHashMap.get(hashKeyToLookFor);

                    if(resFromHash != null)
                    {
                        if (resFromHash.getTime() == eachResult2.getTime())
                        {
                            // no-op
                            //System.out.println("MATCH:" + hashKeyToLookFor );
                            //System.out.println("First Result Set:" + runOneTime.longValue());
                            //System.out.println("Second Result Set:" + result2.getTime());
                            //System.out.println("------");
                        }
                        else
                        {
                            // compute diff here and compare with tolerence
                            double diffPercent = ((double) resFromHash.getTime() - (double) eachResult2.getTime()) / (double) resFromHash.getTime();
                            diffPercent *= 100;

                            //**System.out.println("% Change= " + diff);
                            String testDetails = "Test Case Name\t:" + resFromHash.getName() + "\n" +
                                                   "Test Case Spec\t:" + resFromHash.getCustomArray(j).getName() + "=" +
                                                                        resFromHash.getCustomArray(j).getValue() +  "\n" +
                                                   "Tolerance Spec\t:" + tolerance;

                            // regression
                            if (diffPercent < 0)
                            {
                                diffPercent *= -1.00;
                                if (diffPercent >= 0)
                                {
                                    if (diffPercent > tolerance)
                                    {
                                        diffPercent *= -1.00;
                                        System.out.println("Regression Found! \n" + testDetails + "\n" + "% Deviation\t\t:" + diffPercent);
                                        System.out.println("------");
                                    }
                                }

                            }

                            // improvement
                            if (diffPercent >= 0)
                            {
                                if (diffPercent > tolerance)
                                {
                                    //System.out.println("Improvement Found! \n" + testDetails + "\n" + "% Deviation\t\t:" + diffPercent);
                                    //System.out.println("------");
                                }
                            }
                        }
                    } // end of hashtable look up result null check
                } // end for
            }
            else
            {
                // every result should have a custom element
                System.out.println("Result Set 2, Custom Element Missing!::" + eachResult2.getName());
            }


        }


    }


    public static void main(String[] args) throws Exception
    {
        if((args.length > 3) || args.length < 2)
        {
            System.out.println("Invalid Number of arguments to RunComparator utility!");
            System.out.println("Usage: java RunComparator <path to run output file 1> <path to run output file 2> [<tolerance>]");
            System.exit(0);
        }

        // TODO: More checking here and add a driver ant task for this utility
        String runOne = args[0];
        String runTwo = args[1];
        double tolerance = 5.00;    // default value
        if(args[2] != null)
        {
            tolerance = Double.parseDouble(args[2]);
        }


        RunComparator rc = new RunComparator(runOne, runTwo, tolerance);
        rc.compare();
    }

}