trunk/ruta-ep-textruler/src/main/java/org/apache/uima/ruta/textruler/core/TextRulerToolkit.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.uima.ruta.textruler.core;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;
import java.text.CharacterIterator;
import java.text.StringCharacterIterator;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.uima.UIMAFramework;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.Feature;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.TypeSystem;
import org.apache.uima.cas.impl.XmiCasDeserializer;
import org.apache.uima.cas.impl.XmiCasSerializer;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.resource.metadata.TypeDescription;
import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.apache.uima.ruta.engine.RutaEngine;
import org.apache.uima.ruta.ide.core.builder.RutaProjectUtils;
import org.apache.uima.ruta.textruler.TextRulerPlugin;
import org.apache.uima.ruta.textruler.core.TextRulerTarget.MLTargetType;
import org.apache.uima.util.XMLInputSource;
import org.apache.uima.util.XMLSerializer;
import org.eclipse.core.runtime.FileLocator;
import org.eclipse.core.runtime.IPath;
import org.eclipse.core.runtime.Path;

/**
 * 
 * This static class provides all kinds of helper methods and constants that are useful for all
 * kinds of stuff in this project.
 */
public class TextRulerToolkit {

  public static final boolean LOGGING_ENABLED = true;

  public static final boolean DEBUG = false;

  public static final String RUTA_ALL_TYPE_NAME = "org.apache.uima.ruta.type.ALL";

  public static final String RUTA_ANY_TYPE_NAME = "org.apache.uima.ruta.type.ANY";

  public static final String RUTA_WORD_TYPE_NAME = "org.apache.uima.ruta.type.W";

  public static final String RUTA_BREAK_TYPE_NAME = "org.apache.uima.ruta.type.BREAK";

  public static final String RUTA_SPACE_TYPE_NAME = "org.apache.uima.ruta.type.SPACE";

  public static final String RUTA_NUM_TYPE_NAME = "org.apache.uima.ruta.type.NUM";

  public static final String RUTA_MARKUP_TYPE_NAME = "org.apache.uima.ruta.type.MARKUP";

  public static final String RUTA_SPECIAL_TYPE_NAME = "org.apache.uima.ruta.type.SPECIAL";

  public static final String RUTA_NBSP_TYPE_NAME = "org.apache.uima.ruta.type.NBSP";

  public static final String LEFT_BOUNDARY_EXTENSION = "START";

  public static final String RIGHT_BOUNDARY_EXTENSION = "END";

  public static void log(String str) {
    if (LOGGING_ENABLED)
      System.out.println(str);
  }

  public static void logIfDebug(String str) {
    if (DEBUG)
      log(str);
  }

  public static void logIf(boolean condition, String str) {
    if (LOGGING_ENABLED && condition)
      System.out.println(str);
  }

  public static URL getResourceURL(String name) {
    return FileLocator.find(TextRulerPlugin.getDefault().getBundle(), new Path(name), null);
  }

  public static AnalysisEngineDescription getAnalysisEngineDescription(String descFile) {
    AnalysisEngineDescription result = null;
    try {
      XMLInputSource in = new XMLInputSource(descFile);
      result = (AnalysisEngineDescription) UIMAFramework.getXMLParser().parseResourceSpecifier(in);
    } catch (Exception e) {
      TextRulerPlugin.error(e);
      result = null;
    }
    return result;
  }

  public static AnalysisEngineDescription getAnalysisEngineDescription(URL fileURL) {
    AnalysisEngineDescription result = null;
    try {
      XMLInputSource in = new XMLInputSource(fileURL);
      result = (AnalysisEngineDescription) UIMAFramework.getXMLParser().parseResourceSpecifier(in);
    } catch (Exception e) {
      TextRulerPlugin.error(e);
      result = null;
    }
    return result;
  }

  public static AnalysisEngine loadAnalysisEngine(AnalysisEngineDescription desc) {
    AnalysisEngine result = null;
    try {
      result = UIMAFramework.produceAnalysisEngine(desc);
    } catch (Exception e) {
      TextRulerPlugin.error(e);
      result = null;
    }
    return result;
  }

  public static void addBoundaryTypes(AnalysisEngineDescription description, String[] slotNames) {
    List<String> list = new ArrayList<String>();
    for (String eachSlot : slotNames) {
      list.add(eachSlot + TextRulerToolkit.LEFT_BOUNDARY_EXTENSION);
      list.add(eachSlot + TextRulerToolkit.RIGHT_BOUNDARY_EXTENSION);
    }
    TypeSystemDescription typeSystem = description.getAnalysisEngineMetaData().getTypeSystem();
    for (String string : list) {
      TypeDescription type = typeSystem.getType(string);
      if (type == null) {
        typeSystem.addType(string, "", "uima.tcas.Annotation");
      }
    }
  }

  public static CAS readCASfromXMIFile(String filename, AnalysisEngine ae, CAS reuseCAS) {
    return readCASfromXMIFile(new File(filename), ae, reuseCAS);
  }

  public static CAS readCASfromXMIFile(File file, AnalysisEngine ae, CAS reuseCAS) {
    FileInputStream inputStream = null;
    try {
      CAS resultCas;
      inputStream = new FileInputStream(file);
      if (reuseCAS != null) {
        reuseCAS.reset();
        resultCas = reuseCAS;
      } else {
        resultCas = GlobalCASSource.allocCAS(ae); // ae.newCAS();
      }
      XmiCasDeserializer.deserialize(inputStream, resultCas, true);
      return resultCas;
    } catch (Exception e) {
      TextRulerPlugin.error(e);
    } finally {
      try {
        if (inputStream != null)
          inputStream.close();
      } catch (Exception e) {
        TextRulerPlugin.error(e);
      }
    }
    return null;
  }

  public static void writeCAStoXMIFile(CAS aCas, String filename)// throws
  // IOException,
  // SAXException
  {
    File newFile = new File(filename);
    FileOutputStream out = null;

    try {
      // write XMI
      out = new FileOutputStream(newFile);
      XmiCasSerializer ser = new XmiCasSerializer(aCas.getTypeSystem());
      XMLSerializer xmlSer = new XMLSerializer(out, false);
      ser.serialize(aCas, xmlSer.getContentHandler());
    } catch (Exception e) {
      TextRulerPlugin.error(e);
    } finally {
      if (out != null) {
        try {
          out.close();
        } catch (Exception e) {
          TextRulerPlugin.error(e);
        }

      }
    }
  }

  public static List<AnnotationFS> extractAnnotationsForSlotName(CAS aCas, String slotName) {
    List<AnnotationFS> result = new ArrayList<AnnotationFS>();
    TypeSystem ts = aCas.getTypeSystem();
    Type slotType = ts.getType(slotName);
    FSIterator<AnnotationFS> it = aCas.getAnnotationIndex(slotType).iterator(true);
    if (!it.isValid()) {
      // System.out.println("##### -> iterator not valid for slots!!");
    }
    while (it.isValid()) {
      AnnotationFS fs = it.get();

      // quick hack for quantifier bug in TM:
      AnnotationFS previous = result.size() > 0 ? result.get(result.size() - 1) : null;
      if (previous == null || previous.getBegin() != fs.getBegin()
              || previous.getEnd() != fs.getEnd())
        result.add(fs);
      else {
        logIfDebug("******** TM QUANTIFIER BUG ?? Multiple annotation: " + fs.getType().getName());
      }
      it.moveToNext();
    }

    return result;
  }

  private static List<AnnotationFS> getAnnotationWithinBounds(CAS aCas, int posStart, int posEnd,
          Set<String> filterSet, Type rootType) {
    List<AnnotationFS> result = new ArrayList<AnnotationFS>();
    TypeSystem ts = aCas.getTypeSystem();
    try {

      // TODO wie in TMs AnnotationRetrieval evtl nicht den subiterator
      // nehmen, da der auf
      // type comparisons basiert, die wir evtl nicht gegeben haben!?
      AnnotationFS boundaryAnnotation = aCas.createAnnotation(ts.getType("uima.tcas.Annotation"),
              posStart > 0 ? posStart - 1 : 0, posEnd); // TODO ist das
      // richtig so??!!
      FSIterator<AnnotationFS> it = aCas.getAnnotationIndex().subiterator(boundaryAnnotation, true,
              true);
      while (it.isValid()) {
        AnnotationFS fs = it.get();
        if (fs.getBegin() < posStart || fs.getEnd() > posEnd) {
          it.moveToNext();
          continue;
        }
        if (rootType != null) {
          if (!ts.subsumes(rootType, fs.getType())) {
            it.moveToNext();
            continue;
          }
        }
        if (filterSet == null || !filterSet.contains(fs.getType().getName())) {
          result.add(fs);
        }

        it.moveToNext();
      }

    } catch (Exception e) {
      TextRulerPlugin.error(e);
    }
    return result;
  }

  public static List<AnnotationFS> getAnnotationsBeforePosition(CAS aCas, int position,
          int maxCount, Set<String> filterSet, Type rootType) {
    List<AnnotationFS> result = getAnnotationWithinBounds(aCas, 0, position, filterSet, rootType);
    if (maxCount > 0) {
      while (result.size() > maxCount)
        result.remove(0); // remove from front of queue !
    }
    return result;
  }

  public static List<AnnotationFS> getAnnotationsAfterPosition(CAS aCas, int position,
          int maxCount, Set<String> filterSet, Type rootType) {
    int maxPos = aCas.getDocumentText().length() - 1;
    List<AnnotationFS> result = getAnnotationWithinBounds(aCas, position, maxPos, filterSet,
            rootType);
    if (maxCount > 0) {
      while (result.size() > maxCount)
        result.remove(result.size() - 1); // remove from end of queue!
    }
    return result;
  }

  public static List<AnnotationFS> getAnnotationsWithinBounds(CAS aCas, int start, int end,
          Set<String> filterSet, Type rootType) {
    return getAnnotationWithinBounds(aCas, start, end, filterSet, rootType);
  }

  public static List<AnnotationFS> getOtherAnnotationsOverToken(CAS aCas,
          AnnotationFS tmTokenAnnotation, Set<String> filterSet) {
    List<AnnotationFS> result = new ArrayList<AnnotationFS>();
    // filter out document annotation!!
    FSIterator<AnnotationFS> it = aCas.getAnnotationIndex().iterator();
    Type tokenType = tmTokenAnnotation.getType();
    FSIterator<AnnotationFS> leftIt = null;
    FSIterator<AnnotationFS> rightIt = null;
    TypeSystem ts = aCas.getTypeSystem();
    Type tmRootType = ts.getType(RUTA_ALL_TYPE_NAME);
    Set<String> allFilters = new HashSet<String>();
    allFilters.add("uima.tcas.DocumentAnnotation");
    allFilters.add(RutaEngine.BASIC_TYPE);
    if (filterSet != null)
      allFilters.addAll(filterSet);
    for (; it.isValid(); it.moveToNext()) {
      AnnotationFS fs = (AnnotationFS) it.get();
      if (fs.getBegin() == tmTokenAnnotation.getBegin()
              && fs.getEnd() == tmTokenAnnotation.getEnd() && fs.getType().equals(tokenType)) {
        leftIt = it;

        rightIt = it.copy();
        break;
      }
    }
    if (leftIt == null)
      return null; // the token annotation was not found !
    if (leftIt.isValid())
      leftIt.moveToPrevious(); // leave our token annotation behind us...
    // search from the token annotation to the left
    for (; leftIt.isValid(); leftIt.moveToPrevious()) {
      AnnotationFS fs = (AnnotationFS) leftIt.get();
      if (fs.getEnd() <= tmTokenAnnotation.getBegin())
        break; // if that happens we are out of reach and can stop
      if (fs.getBegin() <= tmTokenAnnotation.getBegin()
              && fs.getEnd() >= tmTokenAnnotation.getEnd()
              && !allFilters.contains(fs.getType().getName())
              && !ts.subsumes(tmRootType, fs.getType()))
        result.add(fs);
    }

    // search from the token annotation to the right
    if (rightIt.isValid())
      rightIt.moveToNext(); // leave our token annotation behind us...
    for (; rightIt.isValid(); rightIt.moveToNext()) {
      AnnotationFS fs = (AnnotationFS) rightIt.get();
      if (fs.getBegin() >= tmTokenAnnotation.getEnd())
        break; // if that happens we are out of reach and can stop
      if (fs.getBegin() <= tmTokenAnnotation.getBegin()
              && fs.getEnd() >= tmTokenAnnotation.getEnd()
              && !allFilters.contains(fs.getType().getName())
              && !ts.subsumes(tmRootType, fs.getType()))
        result.add(fs);
    }
    return result;
  }

  public static synchronized Set<String> getFilterSetWithSlotNames(String[] slotNames,
          Set<String> otherFilters) {
    Set<String> result = new HashSet<String>(otherFilters);
    result.add(RutaEngine.BASIC_TYPE);
    if (slotNames != null)
      for (String s : slotNames)
        result.add(s);
    return result;
  }

  public static synchronized Set<String> getFilterSetWithSlotName(String slotName,
          Set<String> otherFilters) {
    String[] sn = { slotName };
    return getFilterSetWithSlotNames(sn, otherFilters);
  }

  public static synchronized String getStandardFilterSetString() {
    String str = "";
    for (String s : getStandardFilterSet(null))
      if (str.length() == 0)
        str += s;
      else
        str += ", " + s;
    return str;
  }

  public static synchronized Set<String> getStandardFilterSet(String[] slotNames) {
    Set<String> filterSet = new HashSet<String>();
    if (slotNames != null) {
      for (String s : slotNames)
        filterSet.add(s);
    }
    filterSet.add(RUTA_SPACE_TYPE_NAME);
    filterSet.add(RUTA_BREAK_TYPE_NAME);
    filterSet.add(RUTA_MARKUP_TYPE_NAME);
    filterSet.add(RUTA_NBSP_TYPE_NAME);
    return filterSet;
  }

  public static synchronized Set<String> getStandardFeatureFilterSet() {
    Set<String> filterSet = new HashSet<String>();

    filterSet.add("uima.cas.AnnotationBase:sofa");
    filterSet.add("uima.tcas.Annotation:begin");
    filterSet.add("uima.tcas.Annotation:end");
    filterSet.add("org.apache.uima.ruta.type.RutaBasic:Replacement");
    return filterSet;
  }

  // return the example of the list if found, null otherwise
  public static synchronized TextRulerExample exampleListContainsAnnotation(
          List<TextRulerExample> list, TextRulerAnnotation ann) {
    TextRulerExample needle = new TextRulerExample(null, ann, true, null);
    int index = Collections.binarySearch(list, needle, new Comparator<TextRulerExample>() {
      public int compare(TextRulerExample o1, TextRulerExample o2) {
        TextRulerAnnotation afs1 = o1.getAnnotation();
        TextRulerAnnotation afs2 = o2.getAnnotation();
        if (afs1.getBegin() < afs2.getBegin())
          return -1;
        else if (afs1.getBegin() > afs2.getBegin())
          return 1;
        else if (afs1.getEnd() > afs2.getEnd())
          return -1;
        else if (afs1.getEnd() < afs2.getEnd())
          return 1;
        else
          return 0;
      }
    });
    if (index >= 0)
      return list.get(index);
    else
      return null;
  }

  public static synchronized String addTrailingSlashToPath(String path) {
    if (!(path.endsWith("/") || path.endsWith("\\")))
      path = path + System.getProperty("file.separator");
    return path;
  }

  public static synchronized String createTemporaryDirectory() throws IOException {

    final File temp;

    temp = File.createTempFile("temp", Long.toString(System.nanoTime()));
    if (!(temp.delete()))
      return null;
    if (!(temp.mkdir()))
      return null;
    temp.deleteOnExit();
    return addTrailingSlashToPath(temp.getPath());
  }

  public static synchronized String getTypeShortName(String typeName) {
    if (typeName.indexOf(".") >= 0) {
      String components[] = typeName.split("\\.");
      return components[components.length - 1];
    } else
      return typeName;
  }

  public static synchronized String getEngineDescriptorFromTMSourceFile(IPath scriptFilePath) {
    IPath folder = scriptFilePath;

    while (!folder.lastSegment().equals(RutaProjectUtils.getDefaultScriptLocation())) {
      folder = folder.removeLastSegments(1);
    }
    IPath relativeTo = scriptFilePath.makeRelativeTo(folder);
    IPath projectPath = folder.removeLastSegments(1);
    String elementName = scriptFilePath.lastSegment();
    int lastIndexOf = elementName.lastIndexOf(RutaEngine.SCRIPT_FILE_EXTENSION);
    if (lastIndexOf != -1) {
      elementName = elementName.substring(0, lastIndexOf);
    }
    IPath descPath = projectPath.append(RutaProjectUtils.getDefaultDescriptorLocation());
    IPath descPackagePath = descPath.append(relativeTo.removeLastSegments(1));
    return descPackagePath.append(elementName + "Engine.xml").toString();
  }

  public static synchronized String getTypeSystemDescriptorFromTMSourceFile(IPath scriptFilePath) {
    IPath folder = scriptFilePath;

    while (!folder.lastSegment().equals(RutaProjectUtils.getDefaultScriptLocation())) {
      folder = folder.removeLastSegments(1);
    }
    IPath relativeTo = scriptFilePath.makeRelativeTo(folder);
    IPath projectPath = folder.removeLastSegments(1);
    String elementName = scriptFilePath.lastSegment();
    int lastIndexOf = elementName.lastIndexOf(RutaEngine.SCRIPT_FILE_EXTENSION);
    if (lastIndexOf != -1) {
      elementName = elementName.substring(0, lastIndexOf);
    }
    IPath descPath = projectPath.append(RutaProjectUtils.getDefaultDescriptorLocation());
    IPath descPackagePath = descPath.append(relativeTo.removeLastSegments(1));
    return descPackagePath.append(elementName + "TypeSystem.xml").toString();
  }

  public static synchronized String escapeForRegExp(String aRegexFragment) {
    final StringBuilder result = new StringBuilder();

    final StringCharacterIterator iterator = new StringCharacterIterator(aRegexFragment);
    char character = iterator.current();
    while (character != CharacterIterator.DONE) {
      /*
       * All literals need to have backslashes doubled.
       */
      if (character == '.') {
        result.append("\\.");
      } else if (character == '\\') {
        result.append("\\\\");
      } else if (character == '?') {
        result.append("\\?");
      } else if (character == '*') {
        result.append("\\*");
      } else if (character == '+') {
        result.append("\\+");
      } else if (character == '&') {
        result.append("\\&");
      } else if (character == ':') {
        result.append("\\:");
      } else if (character == '{') {
        result.append("\\{");
      } else if (character == '}') {
        result.append("\\}");
      } else if (character == '[') {
        result.append("\\[");
      } else if (character == ']') {
        result.append("\\]");
      } else if (character == '(') {
        result.append("\\(");
      } else if (character == ')') {
        result.append("\\)");
      } else if (character == '^') {
        result.append("\\^");
      } else if (character == '$') {
        result.append("\\$");
      } else {
        // the char is not a special one
        // add it to the result as is
        result.append(character);
      }
      character = iterator.next();
    }
    return result.toString();
  }

  public static synchronized String escapeForTMStringParameter(String aTMStringFragment) {
    final StringBuilder result = new StringBuilder();

    final StringCharacterIterator iterator = new StringCharacterIterator(aTMStringFragment);
    char character = iterator.current();
    while (character != CharacterIterator.DONE) {
      if (character == '"') {
        result.append("\\\"");
      } else if (character == '\\') {
        result.append("\\\\");
      } else {
        result.append(character);
      }
      character = iterator.next();
    }
    return result.toString();
  }

  public static synchronized void appendStringToFile(String fileName, String str) {
    try {
      File f = new File(fileName);
      BufferedWriter output;
      if (!f.exists())
        output = new BufferedWriter(new FileWriter(fileName));
      else
        output = new BufferedWriter(new FileWriter(fileName, true));
      output.append(str);
      output.close();
    } catch (IOException e) {
      TextRulerPlugin.error(e);
    }
  }

  public static synchronized TextRulerAnnotation convertToTargetAnnotation(AnnotationFS fs,
          TextRulerExampleDocument doc, TextRulerTarget target, TypeSystem ts) {
    AnnotationFS theAnnotation;
    if (target.type == MLTargetType.SINGLE_LEFT_BOUNDARY)
      theAnnotation = fs.getCAS().createAnnotation(ts.getType(target.getSingleSlotTypeName()),
              fs.getBegin(), fs.getBegin());
    else if (target.type == MLTargetType.SINGLE_RIGHT_BOUNDARY)
      theAnnotation = fs.getCAS().createAnnotation(ts.getType(target.getSingleSlotTypeName()),
              fs.getEnd(), fs.getEnd());
    else
      theAnnotation = fs;
    return new TextRulerAnnotation(theAnnotation, doc);
  }

  public static synchronized List<Feature> getFilteredAnnotationFeatures(AnnotationFS afs) {
    List<Feature> result = new ArrayList<Feature>();
    List<Feature> theFeatures = afs.getType().getFeatures();
    Set<String> filters = getStandardFeatureFilterSet();
    for (Feature f : theFeatures)
      if (!filters.contains(f.getName()))
        result.add(f);
    return result;
  }
}