uimaj-2.3.0-incubating/uimaj-component-test-util/src/main/java/org/apache/uima/test/junit_extension/AnnotatorPerformanceTester.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.uima.test.junit_extension;

import java.io.File;
import java.io.FileFilter;
import java.util.HashMap;
import java.util.logging.LogManager;

import junit.framework.Assert;

import org.apache.uima.UIMAFramework;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.cas.CAS;
import org.apache.uima.internal.util.Timer;
import org.apache.uima.resource.ResourceManager;
import org.apache.uima.resource.ResourceSpecifier;
import org.apache.uima.util.FileUtils;
import org.apache.uima.util.Level;
import org.apache.uima.util.Logger;
import org.apache.uima.util.XMLInputSource;

/**
 * AnnotatorPerfTester is a helper class to execute annotator performance tests. The performance
 * test results are returned as {@link PerformanceTestResultImpl} object.
 * 
 */
public class AnnotatorPerformanceTester {

  private static class FileFileFilter implements FileFilter {

    private FileFileFilter() {
      super();
    }

    public boolean accept(File arg0) {
      return arg0.isFile();
    }

  }

  private static HashMap logLevels = new HashMap(9);
  static {
    logLevels.put("OFF", Level.OFF);
    logLevels.put("SEVERE", Level.SEVERE);
    logLevels.put("WARNING", Level.WARNING);
    logLevels.put("INFO", Level.INFO);
    logLevels.put("CONFIG", Level.CONFIG);
    logLevels.put("FINE", Level.FINE);
    logLevels.put("FINER", Level.FINER);
    logLevels.put("FINEST", Level.FINEST);
    logLevels.put("ALL", Level.ALL);
  }

  /**
   * runs an annotator performance test
   * 
   * @param repeatSingle
   *          if true, every document is process "numsToRun" times before the next document is
   *          processed. If false, all documents are processed and this is repeated "numsToRun"
   *          times.
   * 
   * @param numsToRun
   *          repeat count for the input documents
   * @param taeDescFilePath
   *          ae descriptor - absolute file path
   * @param testFileDir
   *          test file directory
   * @param dataPath
   *          ae datapath
   * @param doWarmup
   *          do warum for analysis engine - runs an short english sample document
   * @return PerformanceTestResult - returns the performance test results
   * 
   * @throws Exception
   */
  public static PerformanceTestResult runPerformanceTest(boolean repeatSingle, int numsToRun,
          File taeDescFilePath, File testFileDir, String dataPath, boolean doWarmup)
          throws Exception {

    // create performance result object
    PerformanceTestResultImpl result = new PerformanceTestResultImpl();

    // check mandetory settings
    Assert.assertNotNull(taeDescFilePath);
    Assert.assertNotNull(testFileDir);

    // save settings
    result.setRepeatSingleMode(repeatSingle);
    result.setDoWarmup(doWarmup);
    result.setNumsToRun(numsToRun);
    result.setAeDescFilePath(taeDescFilePath);
    result.setTestFileDir(testFileDir);
    result.setDatapath(dataPath);

    // set and check test file directory
    if (testFileDir == null || !testFileDir.isDirectory() || !testFileDir.canRead()) {
      throw new Exception("test file directory not valid");
    }

    // get current log level setting
    Level defaultLogLevel = (Level) logLevels.get(LogManager.getLogManager()
            .getProperty(".level"));

    if (defaultLogLevel == null) {
      // no log level was specified, use default log level settings "INFO" that is also
      // used by the Java logging framework.
      defaultLogLevel = Level.INFO;
    }
    // turn of logging for the performance test
    Logger logger = UIMAFramework.getLogger();
    logger.setLevel(Level.OFF);

    //create timer 
    Timer globalTimer = new Timer();
    Timer initTimer = new Timer();
    Timer warmupTimer = new Timer();
    Timer ioTimer = new Timer();
    Timer processResetTimer = new Timer();
    Timer cleanupTimer = new Timer();
    Timer documentPreparationTimer = new Timer();
    
    //start timer for global time
    globalTimer.start();

    // init analysis engine
    try {

      // start initialization timer   
      initTimer.start();
      
      // set datapath
      ResourceManager resMgr = UIMAFramework.newDefaultResourceManager();
      if (dataPath != null) {
        resMgr.setDataPath(dataPath);
      }

      AnalysisEngine ae = null;
      CAS cas = null;
      // get resource specifier from XML file
      XMLInputSource in = new XMLInputSource(taeDescFilePath);
      ResourceSpecifier specifier = UIMAFramework.getXMLParser().parseResourceSpecifier(in);

      // create analysis engine with resource manager
      ae = UIMAFramework.produceAnalysisEngine(specifier, resMgr, null);
      // check ae
      Assert.assertNotNull(ae);

      // create new cas
      cas = ae.newCAS();
      // check cas
      Assert.assertNotNull(cas);

      // access cas type system
      cas.getTypeSystem().getFeatureByFullName(CAS.FEATURE_FULL_NAME_LANGUAGE);

      // stop initalization timer
      initTimer.stop();
      result.setInitTime(initTimer.getTimeSpan());

      if (doWarmup) {
        // start warmup timer     
        warmupTimer.start();

        // process dummy document
        cas.setDocumentLanguage("en");
        cas.setDocumentText("This is a test sentence.");
        ae.process(cas);
        cas.reset();

        // stop warmup timer
        warmupTimer.stop();
        result.setWarmupTime(warmupTimer.getTimeSpan());
      }

      // start io timer
      ioTimer.start();

      // read all files in the test file directory
      File[] inputFiles = testFileDir.listFiles(new FileFileFilter());
      // create string array for the file content and language
      String[] fileTexts = new String[inputFiles.length];
      String[] languages = new String[inputFiles.length];
      int numChars = 0;
      long fileSize = 0;
      // iterate of all input files and extract content and language
      for (int i = 0; i < inputFiles.length; i++) {
        // get file language
        languages[i] = inputFiles[i].getName().substring(0, 2);
        // get file content
        fileTexts[i] = FileUtils.file2String(inputFiles[i], "UTF-8");
        fileSize += inputFiles[i].length();
        // count characters
        numChars += fileTexts[i].length();
      }

      // stop io timer
      ioTimer.stop();

      // save results
      result.setNumberOfFiles(inputFiles.length);
      result.setNumberOfCharacters(numChars);
      result.setTotalFileSize(fileSize);
      result.setIoTime(ioTimer.getTimeSpan());

      // start real processing
      int numAnnot = 0;

      // check repeat single mode setting
      // repeatSingle=true: iterates of all files and repeat each file "numsToRun" times
      // repeatSingle=false: iterates of all files and repeat the collection "numsToRun" times
      if (repeatSingle) {
        // iterate over all text files (over the cached content)
        for (int i = 0; i < fileTexts.length; i++) {
          // file repeat mode
          // iterate over the current document "numsToRun" times
          for (int j = 0; j < numsToRun; j++) {
            documentPreparationTimer.start();
            // set cas data
            cas.setDocumentLanguage(languages[i]);
            cas.setDocumentText(fileTexts[i]);
            documentPreparationTimer.stop();
            processResetTimer.start();
            ae.process(cas);
            processResetTimer.stop();
            documentPreparationTimer.start();
            numAnnot += cas.getAnnotationIndex().size();
            cas.reset();
            documentPreparationTimer.stop();
          }
        }
      }
      // use collection repeat mode
      else {
        // process the file collection "numsToRun" times
        for (int j = 0; j < numsToRun; j++) {
          // iterate over all text files (over the cached content)
          for (int i = 0; i < fileTexts.length; i++) {
            documentPreparationTimer.start();
            // set cas data
            cas.setDocumentLanguage(languages[i]);
            cas.setDocumentText(fileTexts[i]);
            documentPreparationTimer.stop();
            processResetTimer.start();
            ae.process(cas);
            processResetTimer.stop();
            documentPreparationTimer.start();
            numAnnot += cas.getAnnotationIndex().size();
            cas.reset();
            documentPreparationTimer.stop();
          }
        }
      }

      // cleanup ae and stop global timer
      cleanupTimer.start();
      ae.destroy();
      ae = null;
      cleanupTimer.stop();
      globalTimer.stop();

      // save results
      result.setNumberOfCreatedAnnotations(numAnnot);
      result.setOverallTime(globalTimer.getTimeSpan());
      result.setProcessingTime(processResetTimer.getTimeSpan());
      result.setCleanupTime(cleanupTimer.getTimeSpan());
      result.setDocumentPreparationTime(documentPreparationTimer.getTimeSpan());

      // turn on logging as it was before
      logger.setLevel(defaultLogLevel);

      // return result object
      return result;

    } catch (Exception e) { // Bail out.
      throw e;
    }

  }

}