| /* | |
| * Licensed to the Apache Software Foundation (ASF) under one | |
| * or more contributor license agreements. See the NOTICE file | |
| * distributed with this work for additional information | |
| * regarding copyright ownership. The ASF licenses this file | |
| * to you under the Apache License, Version 2.0 (the | |
| * "License"); you may not use this file except in compliance | |
| * with the License. You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, | |
| * software distributed under the License is distributed on an | |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| * KIND, either express or implied. See the License for the | |
| * specific language governing permissions and limitations | |
| * under the License. | |
| */ | |
| package org.apache.uima.ruta.cde.ui; | |
| import java.io.File; | |
| import java.io.FileInputStream; | |
| import java.net.URL; | |
| import java.util.ArrayList; | |
| import java.util.HashMap; | |
| import java.util.List; | |
| import org.apache.commons.lang3.StringUtils; | |
| import org.apache.uima.UIMAFramework; | |
| import org.apache.uima.cas.CAS; | |
| import org.apache.uima.cas.Type; | |
| import org.apache.uima.cas.TypeSystem; | |
| import org.apache.uima.cas.impl.XmiCasDeserializer; | |
| import org.apache.uima.resource.metadata.FsIndexDescription; | |
| import org.apache.uima.resource.metadata.TypePriorities; | |
| import org.apache.uima.resource.metadata.TypeSystemDescription; | |
| import org.apache.uima.ruta.addons.RutaAddonsPlugin; | |
| import org.apache.uima.ruta.cde.IRutaConstraint; | |
| import org.apache.uima.ruta.cde.IRutaRuleConstraint; | |
| import org.apache.uima.ruta.cde.utils.ConstraintData; | |
| import org.apache.uima.ruta.cde.utils.DocumentData; | |
| import org.apache.uima.ruta.cde.utils.EvaluationMeasures; | |
| import org.apache.uima.ruta.testing.evaluator.ICasEvaluator; | |
| import org.apache.uima.ruta.testing.preferences.TestingPreferenceConstants; | |
| import org.apache.uima.util.CasCreationUtils; | |
| import org.apache.uima.util.XMLInputSource; | |
| import org.eclipse.core.commands.ExecutionEvent; | |
| import org.eclipse.core.commands.ExecutionException; | |
| import org.eclipse.core.commands.IHandler; | |
| import org.eclipse.core.commands.IHandlerListener; | |
| import org.eclipse.core.runtime.IProgressMonitor; | |
| import org.eclipse.core.runtime.IStatus; | |
| import org.eclipse.core.runtime.Status; | |
| import org.eclipse.core.runtime.jobs.IJobChangeEvent; | |
| import org.eclipse.core.runtime.jobs.Job; | |
| import org.eclipse.core.runtime.jobs.JobChangeAdapter; | |
| import org.eclipse.jface.preference.IPreferenceStore; | |
| import org.eclipse.ui.PartInitException; | |
| import org.eclipse.ui.handlers.HandlerUtil; | |
| public class DocumentViewRunHandler implements IHandler { | |
| private class DocumentViewRunJobChangeAdapter extends JobChangeAdapter { | |
| // Composite of the ConstraintSelectView | |
| private DocumentSelectComposite composite; | |
| DocumentViewRunJobChangeAdapter(DocumentSelectComposite composite) { | |
| super(); | |
| this.composite = composite; | |
| } | |
| @Override | |
| public void done(IJobChangeEvent event) { | |
| if (event.getResult().isOK()) { | |
| updateUI(composite); | |
| } | |
| } | |
| } | |
| private void updateUI(final DocumentSelectComposite composite) { | |
| composite.getDisplay().asyncExec(new Runnable() { | |
| public void run() { | |
| try { | |
| composite.getViewer().refresh(); | |
| ArrayList<Double[]> results = new ArrayList<Double[]>(); | |
| if (composite.getTestDataPath() != null) { | |
| for (DocumentData data : composite.getDocumentList()) { | |
| double result = data.getAugmentedResult(); | |
| double score = data.getFMeasure(); | |
| results.add(new Double[] { result, score }); | |
| } | |
| } | |
| composite.updateMeasureReport("" + EvaluationMeasures.getMeasureReport(results)); | |
| } catch (Exception e) { | |
| RutaAddonsPlugin.error(e); | |
| } | |
| composite.getViewer().refresh(); | |
| composite.update(); | |
| } | |
| }); | |
| } | |
| private class DocumentViewRunHandlerJob extends Job { | |
| private List<ConstraintData> constraints; | |
| private List<DocumentData> documents; | |
| private File typeSystem; | |
| private String testDataPath; | |
| private final DocumentSelectComposite composite; | |
| DocumentViewRunHandlerJob(ExecutionEvent event, List<ConstraintData> constraints, | |
| ArrayList<DocumentData> documents, File typeSystem, String testDataPath, | |
| DocumentSelectComposite composite) { | |
| super("UIMA Ruta CDE"); | |
| this.constraints = constraints; | |
| this.documents = documents; | |
| this.typeSystem = typeSystem; | |
| this.testDataPath = testDataPath; | |
| this.composite = composite; | |
| } | |
| public IStatus run(IProgressMonitor monitor) { | |
| monitor.beginTask("UIMA Ruta CDE", constraints.size() * documents.size()); | |
| CAS inputCAS = null; | |
| CAS testCas = null; | |
| try { | |
| URL tpUrl = this.getClass().getResource("/org/apache/uima/ruta/engine/TypePriorities.xml"); | |
| TypePriorities typePriorities = UIMAFramework.getXMLParser().parseTypePriorities( | |
| new XMLInputSource(tpUrl)); | |
| TypeSystemDescription descriptor = (TypeSystemDescription) UIMAFramework.getXMLParser() | |
| .parse(new XMLInputSource(typeSystem)); | |
| inputCAS = CasCreationUtils | |
| .createCas(descriptor, typePriorities, new FsIndexDescription[0]); | |
| testCas = CasCreationUtils.createCas(descriptor, null, new FsIndexDescription[0]); | |
| } catch (Exception e) { | |
| RutaAddonsPlugin.error(e); | |
| } | |
| for (DocumentData document : documents) { | |
| if (monitor.isCanceled()) { | |
| return Status.CANCEL_STATUS; | |
| } | |
| double count = 0; | |
| double augResult = 0; | |
| document.setResults(new ArrayList<String[]>()); | |
| if (constraints.size() > 0) { | |
| monitor.setTaskName("UIMA Ruta CDE: " + document.getDocument().getName()); | |
| try { | |
| inputCAS.reset(); | |
| XmiCasDeserializer.deserialize(new FileInputStream(document.getDocument()), inputCAS, | |
| true); | |
| for (ConstraintData constraintData : constraints) { | |
| if (monitor.isCanceled()) { | |
| return Status.CANCEL_STATUS; | |
| } | |
| String[] partialResult = new String[2]; | |
| IRutaConstraint constraint = constraintData.getConstraint(); | |
| if (constraint instanceof IRutaRuleConstraint) { | |
| ((IRutaRuleConstraint) constraint).setTypeSystemLocation(typeSystem | |
| .getAbsolutePath()); | |
| } | |
| // Calculating and adding results to the | |
| // documentData object | |
| Double partResult = constraint.processConstraint(inputCAS); | |
| if (partResult != null) { | |
| partialResult[0] = constraint.getDescription(); | |
| partialResult[1] = String.valueOf(partResult); | |
| int weight = constraintData.getWeight(); | |
| count += weight; | |
| double weightedResult = partResult * weight; | |
| augResult += weightedResult; | |
| // inputCAS.release(); | |
| document.getResults().add(partialResult); | |
| } | |
| monitor.worked(1); | |
| } | |
| if (testDataPath != null) { | |
| // Hashmap to save all the FileNames without | |
| // fileExtension and FullPath, both as | |
| // strings | |
| HashMap<String, String> testFilesMap = createFileMap(testDataPath); | |
| String documentFileName = getFileNameWithoutExtensions(document.getDocument() | |
| .getAbsolutePath()); | |
| if (testFilesMap.get(documentFileName) != null) { | |
| File testFile = new File(testFilesMap.get(documentFileName)); | |
| testCas.reset(); | |
| XmiCasDeserializer.deserialize(new FileInputStream(testFile), testCas, true); | |
| IPreferenceStore store = RutaAddonsPlugin.getDefault().getPreferenceStore(); | |
| String factoryName = store.getString(TestingPreferenceConstants.EVALUATOR_FACTORY); | |
| boolean includeSubtypes = store.getBoolean(TestingPreferenceConstants.INCLUDE_SUBTYPES); | |
| boolean useAllTypes = store.getBoolean(TestingPreferenceConstants.ALL_TYPES); | |
| ICasEvaluator evaluator = RutaAddonsPlugin.getCasEvaluatorFactoryById(factoryName) | |
| .createEvaluator(); | |
| ArrayList<String> excludedTypes = new ArrayList<String>(); | |
| CAS resultCas = evaluator.evaluate(testCas, inputCAS, excludedTypes, includeSubtypes, useAllTypes); | |
| document.setFMeasure(calculateF1(resultCas)); | |
| } | |
| } | |
| } catch (Exception e) { | |
| RutaAddonsPlugin.error(e); | |
| } | |
| } | |
| if (count != 0) { | |
| augResult = augResult / count; | |
| } | |
| document.setAugmentedResult(augResult); | |
| updateUI(composite); | |
| } | |
| inputCAS.release(); | |
| testCas.release(); | |
| monitor.done(); | |
| return Status.OK_STATUS; | |
| } | |
| } | |
| public void addHandlerListener(IHandlerListener arg0) { | |
| } | |
| public void dispose() { | |
| } | |
| public Object execute(ExecutionEvent event) throws ExecutionException { | |
| ConstraintSelectView constraintView; | |
| try { | |
| constraintView = (ConstraintSelectView) HandlerUtil.getActiveWorkbenchWindow(event) | |
| .getWorkbench().getActiveWorkbenchWindow().getActivePage() | |
| .showView("org.apache.uima.ruta.cde.ui.ConstraintSelectView"); | |
| ConstraintSelectComposite composite = (ConstraintSelectComposite) constraintView | |
| .getComposite(); | |
| List<ConstraintData> constraintList = composite.getConstraintList(); | |
| DocumentView docView = (DocumentView) HandlerUtil.getActiveWorkbenchWindow(event) | |
| .getWorkbench().getActiveWorkbenchWindow().getActivePage() | |
| .showView("org.apache.uima.ruta.cde.ui.DocumentView"); | |
| DocumentSelectComposite docComposite = docView.getDocComposite(); | |
| ArrayList<DocumentData> documents = docComposite.getDocumentList(); | |
| if (documents.isEmpty()) { | |
| docComposite.setDocumentsByDir(); | |
| } | |
| File typeSystem = docComposite.getTypeSystem(); | |
| DocumentViewRunHandlerJob job = new DocumentViewRunHandlerJob(event, constraintList, | |
| documents, typeSystem, docComposite.getTestDataPath(), docComposite); | |
| job.addJobChangeListener(new DocumentViewRunJobChangeAdapter(docComposite)); | |
| job.schedule(); | |
| } catch (PartInitException e) { | |
| RutaAddonsPlugin.error(e); | |
| } | |
| return null; | |
| } | |
| public boolean isEnabled() { | |
| return true; | |
| } | |
| public boolean isHandled() { | |
| return true; | |
| } | |
| public void removeHandlerListener(IHandlerListener arg0) { | |
| } | |
| private double calculateF1(CAS resultCas) { | |
| TypeSystem ts = resultCas.getTypeSystem(); | |
| Type falsePositiveType = ts.getType(ICasEvaluator.FALSE_POSITIVE); | |
| Type falseNegativeType = ts.getType(ICasEvaluator.FALSE_NEGATIVE); | |
| Type truePositiveType = ts.getType(ICasEvaluator.TRUE_POSITIVE); | |
| int falsePositiveCount = resultCas.getAnnotationIndex(falsePositiveType).size(); | |
| int falseNegativeCount = resultCas.getAnnotationIndex(falseNegativeType).size(); | |
| int truePositiveCount = resultCas.getAnnotationIndex(truePositiveType).size(); | |
| double a = falsePositiveCount; | |
| double b = falseNegativeCount; | |
| double c = truePositiveCount; | |
| double precision = c / (c + a); | |
| double recall = c / (c + b); | |
| double fMeasure = 2 * (precision * recall) / (precision + recall); | |
| fMeasure = fMeasure * 10000; | |
| fMeasure = Math.round(fMeasure); | |
| fMeasure = fMeasure / 10000; | |
| return fMeasure; | |
| } | |
| public HashMap<String, String> createFileMap(String directoryFolderPath) { | |
| HashMap<String, String> filesMap = new HashMap<String, String>(); | |
| File folder = new File(directoryFolderPath); | |
| if (folder.isDirectory()) { | |
| String[] files = folder.list(null); | |
| for (String fileName : files) { | |
| String fullPath = folder + System.getProperty("file.separator") + fileName; | |
| String fileNameWithoutExtension = getFileNameWithoutExtensions(fullPath); | |
| if (!StringUtils.isBlank(fileNameWithoutExtension)) { | |
| filesMap.put(fileNameWithoutExtension, fullPath); | |
| } | |
| } | |
| } | |
| return filesMap; | |
| } | |
| public String getFileNameWithoutExtensions(String path) { | |
| File f = new File(path); | |
| String fileNameNoExtension = ""; | |
| // Check if the file for the given Path exists on the systems filesystem | |
| // and | |
| // if it is a file | |
| if (f.exists() && f.isFile()) { | |
| // get the filename between the last separator and the file | |
| // extension | |
| if (path.contains(".")) { | |
| int beginOfFileName = path.lastIndexOf(System.getProperty("file.separator")) + 1; | |
| int endOfFileName = path.indexOf("."); | |
| fileNameNoExtension = path.substring(beginOfFileName, endOfFileName); | |
| } | |
| // if there is no file extension | |
| else { | |
| int beginOfFileName = path.lastIndexOf(System.getProperty("file.separator")); | |
| fileNameNoExtension = path.substring(beginOfFileName); | |
| } | |
| } | |
| return fileNameNoExtension; | |
| } | |
| } |