| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| package org.apache.uima.ruta.textruler.learner.whisk.generic; |
| |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| |
| import org.apache.commons.lang3.StringUtils; |
| import org.apache.uima.cas.CAS; |
| import org.apache.uima.cas.FSIterator; |
| import org.apache.uima.cas.FeatureStructure; |
| import org.apache.uima.cas.Type; |
| import org.apache.uima.cas.TypeSystem; |
| import org.apache.uima.cas.text.AnnotationFS; |
| import org.apache.uima.ruta.textruler.core.TextRulerAnnotation; |
| import org.apache.uima.ruta.textruler.core.TextRulerBasicLearner; |
| import org.apache.uima.ruta.textruler.core.TextRulerExample; |
| import org.apache.uima.ruta.textruler.core.TextRulerExampleDocument; |
| import org.apache.uima.ruta.textruler.core.TextRulerRule; |
| import org.apache.uima.ruta.textruler.core.TextRulerRuleItem; |
| import org.apache.uima.ruta.textruler.core.TextRulerRuleList; |
| import org.apache.uima.ruta.textruler.core.TextRulerRulePattern; |
| import org.apache.uima.ruta.textruler.core.TextRulerSlotPattern; |
| import org.apache.uima.ruta.textruler.core.TextRulerStatisticsCollector; |
| import org.apache.uima.ruta.textruler.core.TextRulerTarget; |
| import org.apache.uima.ruta.textruler.core.TextRulerToolkit; |
| import org.apache.uima.ruta.textruler.core.TextRulerWordConstraint; |
| import org.apache.uima.ruta.textruler.extension.TextRulerLearnerDelegate; |
| import org.apache.uima.ruta.textruler.learner.whisk.generic.WhiskRuleItem.MLWhiskOtherConstraint; |
| |
| public class Whisk extends TextRulerBasicLearner { |
| |
| public final static String WINDOWSIZE_KEY = "windowSize"; |
| |
| public final static String ERROR_THRESHOLD_KEY = "errorThreshold"; |
| |
| public final static String POSTAG_ROOTTYPE_KEY = "posTagRootType"; |
| |
| public final static int STANDARD_WINDOWSIZE = 5; |
| |
| public final static float STANDARD_ERROR_THRESHOLD = 0.1f; |
| |
| public final static String STANDARD_POSTAG_ROOTTYPE = "org.apache.uima.ml.ML.postag"; |
| |
| public static final String CONSIDERED_FEATURES = "consideredFeatures"; |
| |
| public static final String STANDARD_CONSIDERED_FEATURES = ""; |
| |
| TextRulerRuleList ruleList; |
| |
| protected Set<TextRulerExample> coveredExamples; |
| |
| protected int windowSize = STANDARD_WINDOWSIZE; |
| |
| protected double errorThreshold = STANDARD_ERROR_THRESHOLD; |
| |
| protected String posTagRootTypeName = STANDARD_POSTAG_ROOTTYPE; |
| |
| int roundNumber = 0; |
| |
| int allExamplesCount = 0; |
| |
| private List<String> consideredFeatures = new ArrayList<String>(); |
| |
| private Map<String, TextRulerStatisticsCollector> cachedTestedRuleStatistics = new HashMap<String, TextRulerStatisticsCollector>(); |
| |
| public Whisk(String inputDir, String prePropTmFile, String tmpDir, String[] slotNames, |
| Set<String> filterSet, boolean skip, TextRulerLearnerDelegate delegate) { |
| super(inputDir, prePropTmFile, tmpDir, slotNames, filterSet, skip, delegate); |
| // useDynamicAnchoring = true; |
| } |
| |
| @Override |
| public boolean collectNegativeCoveredInstancesWhenTesting() { |
| return false; |
| } |
| |
| @Override |
| protected void doRun() { |
| |
| // we don't use the same overall structure like the original WHISK since |
| // we do not |
| // repeat the whole process for some new training documents at the |
| // user's request, we |
| // learn like the other algorithms from the whole training set, so we |
| // for example do not |
| // need to test the intermediate rule base on a newly "incoming" |
| // training document since we |
| // tested all rules already on all training documents ! |
| |
| // this version of whisk is not tested for mutli slot learning since the |
| // seminar announcements |
| // are not quite suitable for this task: they do not all contain all 4 |
| // slots and some of them |
| // occur more than once in one document ! And the order of them is not |
| // always the same as well! |
| // so this is now made only tested for the single slot case even if it |
| // is built capable of multislot |
| // examples! |
| |
| // this is the inner loop of the WHISK pseudo-code: |
| // For each inst in Training |
| // for each tag |
| |
| cachedTestedRuleStatistics.clear(); |
| ruleList = new TextRulerRuleList(); |
| coveredExamples = new HashSet<TextRulerExample>(); |
| |
| sendStatusUpdateToDelegate("Creating examples...", TextRulerLearnerState.ML_RUNNING, false); |
| for (int i = 0; i < slotNames.length; i++) { |
| TextRulerTarget target = new TextRulerTarget(slotNames[i], this); |
| exampleDocuments.createExamplesForTarget(target); |
| |
| TextRulerExampleDocument[] docs = exampleDocuments.getSortedDocumentsInCacheOptimizedOrder(); |
| |
| allExamplesCount = exampleDocuments.getAllPositiveExamples().size(); |
| |
| for (TextRulerExampleDocument inst : docs) { |
| List<TextRulerExample> tags = inst.getPositiveExamples(); |
| |
| // for each uncovered example -> induce a new rule: |
| for (TextRulerExample tag : tags) { |
| if (!coveredExamples.contains(tag)) { |
| roundNumber++; |
| WhiskRule newRule = growRule(inst, tag); |
| if (shouldAbort()) |
| break; |
| // if (newRule == null) |
| // break; |
| // else |
| if (newRule != null |
| && (newRule.getCoveringStatistics().getCoveredNegativesCount() == 00 || newRule |
| .getLaplacian() <= errorThreshold)) { |
| ruleList.addRule(newRule); |
| coveredExamples.addAll(newRule.getCoveringStatistics().getCoveredPositiveExamples()); |
| sendStatusUpdateToDelegate("New Rule added...", TextRulerLearnerState.ML_RUNNING, |
| true); |
| } |
| } |
| } |
| if (shouldAbort()) |
| return; |
| } |
| } |
| sendStatusUpdateToDelegate("Done", TextRulerLearnerState.ML_DONE, true); |
| cachedTestedRuleStatistics.clear(); |
| } |
| |
| protected WhiskRule growRule(TextRulerExampleDocument doc, TextRulerExample example) { |
| sendStatusUpdateToDelegate("Creating new rule from seed...", TextRulerLearnerState.ML_RUNNING, |
| false); |
| WhiskRule theRule = new WhiskRule(this, example.getTarget(), example); |
| int numberOfSlotsInTag = example.getAnnotations().length; |
| for (int i = 0; i < numberOfSlotsInTag; i++) |
| theRule.getPatterns().add(new TextRulerSlotPattern()); |
| |
| sendStatusUpdateToDelegate("Creating new rule: anchoring...", TextRulerLearnerState.ML_RUNNING, |
| false); |
| for (int i = 0; i < numberOfSlotsInTag; i++) { |
| theRule = anchor(theRule, doc, example, i); |
| if (shouldAbort()) |
| return null; |
| } |
| |
| sendStatusUpdateToDelegate("Creating new rule: extending...", TextRulerLearnerState.ML_RUNNING, |
| false); |
| if (theRule != null) { |
| double oldLaplacian = theRule.getLaplacian(); |
| int subRoundNumber = 0; |
| |
| // repeat while we still make errors... |
| while (theRule.getCoveringStatistics().getCoveredNegativesCount() > 0) { |
| WhiskRule extendedRule = extendRule(theRule, doc, example, subRoundNumber); |
| if (extendedRule == null) { |
| // this way we get the previous rule |
| // as the best rule... |
| break; |
| } |
| theRule = extendedRule; |
| TextRulerToolkit.log("----------------------------"); |
| TextRulerToolkit.log("BEST EXTENSION IS: " + theRule.getRuleString()); |
| TextRulerToolkit.log("Laplacian: " + theRule.getLaplacian() + " ; " |
| + theRule.getCoveringStatistics()); |
| subRoundNumber++; |
| |
| double newLaplacian = theRule.getLaplacian(); |
| if (newLaplacian >= oldLaplacian) { |
| break; |
| } |
| oldLaplacian = newLaplacian; |
| } |
| TextRulerToolkit.log("----------------------------"); |
| TextRulerToolkit.log("FINAL RULE IS : " + theRule.getRuleString()); |
| } |
| return theRule; |
| } |
| |
| protected WhiskRule extendRule(WhiskRule rule, TextRulerExampleDocument doc, |
| TextRulerExample example, int subRoundNumber) { |
| WhiskRule bestRule = null; |
| double bestL = 1.0; |
| int bestRuleConstraintPoints = -1; |
| if (rule.getLaplacian() <= errorThreshold) { |
| bestRule = rule; |
| bestL = rule.getLaplacian(); |
| } |
| List<TextRulerRule> rulesToTest = new ArrayList<TextRulerRule>(); |
| |
| // first only add conditions, e.g., for features |
| |
| List<TextRulerSlotPattern> patterns = rule.getPatterns(); |
| for (TextRulerSlotPattern eachPattern : patterns) { |
| for (TextRulerRuleItem item : eachPattern.fillerPattern) { |
| if (item instanceof WhiskRuleItem) { |
| WhiskRuleItem wri = (WhiskRuleItem) item; |
| WhiskRule proposedRule = rule; |
| TextRulerWordConstraint wordConstraint = wri.getWordConstraint(); |
| for (String eachFeature : consideredFeatures) { |
| if (wordConstraint != null) { |
| Map<String, String> featureMap = wordConstraint.getTokenAnnotation().getFeatureMap(); |
| String stringValue = featureMap.get(eachFeature); |
| if (stringValue != null && !wri.getActivatedFeatures().contains(eachFeature)) { |
| wri.activateFeature(eachFeature); |
| WhiskRule proposedRuleF = proposedRule.copy(); |
| wri.deactivateFeature(eachFeature); |
| proposedRuleF.setNeedsCompile(true); |
| if (!rulesToTest.contains(proposedRuleF)) { |
| rulesToTest.add(proposedRuleF); |
| } |
| } |
| } |
| } |
| if (wordConstraint != null && wordConstraint.isRegExpConstraint() && wri.isHideRegExp()) { |
| wri.setHideRegExp(false); |
| WhiskRule proposedRuleF = proposedRule.copy(); |
| wri.setHideRegExp(true); |
| proposedRuleF.setNeedsCompile(true); |
| if (!rulesToTest.contains(proposedRuleF)) { |
| rulesToTest.add(proposedRuleF); |
| } |
| } |
| } |
| } |
| } |
| |
| List<List<WhiskRuleItem>> slotTerms = getTermsWithinBounds( |
| example.getAnnotations()[0].getBegin(), example.getAnnotations()[0].getEnd(), example); |
| List<List<WhiskRuleItem>> windowTerms = getTermsWithinWindow(slotTerms, example, 0); |
| |
| for (List<WhiskRuleItem> eachList : windowTerms) { |
| for (WhiskRuleItem term : eachList) { |
| |
| if (rule.containsTerm(term)) { |
| continue; |
| } |
| |
| WhiskRule proposedRule = createNewRuleByAddingTerm(rule, term); |
| if (proposedRule == null) |
| continue; |
| WhiskRuleItem t = term; |
| |
| if (!rulesToTest.contains(proposedRule)) |
| rulesToTest.add(proposedRule); |
| |
| // add a second version where we add the exact token content if |
| // it is a regexp item: |
| WhiskRule proposedRule2 = proposedRule; |
| if (t.getWordConstraint().isRegExpConstraint()) { |
| t.setHideRegExp(false); |
| WhiskRule proposedRuleF = proposedRule.copy(); |
| t.setHideRegExp(true); |
| proposedRuleF.setNeedsCompile(true); |
| if (!rulesToTest.contains(proposedRuleF)) { |
| rulesToTest.add(proposedRuleF); |
| } |
| } |
| |
| // extend with feature conditions |
| WhiskRule proposedRuleF = null; |
| for (String eachFeature : consideredFeatures) { |
| Map<String, String> featureMap = t.getWordConstraint().getTokenAnnotation() |
| .getFeatureMap(); |
| String stringValue = featureMap.get(eachFeature); |
| if (stringValue != null) { |
| t.activateFeature(eachFeature); |
| proposedRuleF = proposedRule.copy(); |
| t.deactivateFeature(eachFeature); |
| proposedRuleF.setNeedsCompile(true); |
| if (!rulesToTest.contains(proposedRuleF)) { |
| rulesToTest.add(proposedRuleF); |
| } |
| } |
| } |
| |
| // and now, for WHISK performance testing purposes, we also add POS |
| // tags: |
| // this is not very nice code and not dynamic feature capable, but |
| // for testpurposes |
| // in order to test WHISK with PosTag Terms... |
| if (posTagRootTypeName != null && posTagRootTypeName.length() > 0) { |
| TextRulerAnnotation tokenAnnotation = term.getWordConstraint().getTokenAnnotation(); |
| CAS cas = example.getDocumentCAS(); |
| TypeSystem ts = cas.getTypeSystem(); |
| Type posTagsRootType = ts.getType(posTagRootTypeName); |
| if (ts != null) { |
| // POS-Tags created by our test hmm tagger. |
| List<AnnotationFS> posTagAnnotations = TextRulerToolkit.getAnnotationsWithinBounds(cas, |
| tokenAnnotation.getBegin(), tokenAnnotation.getEnd(), null, posTagsRootType); |
| if (posTagAnnotations.size() > 0) { |
| AnnotationFS posTag = posTagAnnotations.get(0); |
| if (posTag.getBegin() == tokenAnnotation.getBegin() |
| && posTag.getEnd() == tokenAnnotation.getEnd()) { |
| TextRulerAnnotation posTagAnnotation = new TextRulerAnnotation(posTag, doc, |
| consideredFeatures); |
| |
| // 1. most specific term with all constraints we |
| // have: |
| WhiskRule proposedRule3 = proposedRule.copy(); |
| WhiskRuleItem t3 = term; |
| t3.addOtherConstraint(new MLWhiskOtherConstraint(tokenAnnotation, posTagAnnotation)); |
| proposedRule3.setNeedsCompile(true); |
| if (!rulesToTest.contains(proposedRule3)) |
| rulesToTest.add(proposedRule3); |
| |
| // 2. the same without the regexp thingy: |
| if (proposedRule2 != null) { |
| WhiskRule proposedRule4 = proposedRule2.copy(); |
| WhiskRuleItem t4 = term; |
| t4.addOtherConstraint(new MLWhiskOtherConstraint(tokenAnnotation, |
| posTagAnnotation)); |
| proposedRule4.setNeedsCompile(true); |
| if (!rulesToTest.contains(proposedRule4)) |
| rulesToTest.add(proposedRule4); |
| } |
| |
| // 3. last but not least: a rule with only the pos |
| // tag constraint: |
| WhiskRule proposedRule5 = proposedRule.copy(); |
| WhiskRuleItem t5 = term; |
| t5.addOtherConstraint(new MLWhiskOtherConstraint(tokenAnnotation, posTagAnnotation)); |
| t5.setWordConstraint(null); |
| proposedRule5.setNeedsCompile(true); |
| if (!rulesToTest.contains(proposedRule5)) { |
| rulesToTest.add(proposedRule5); |
| } |
| |
| } |
| } |
| } |
| } |
| } |
| } |
| if (rulesToTest.size() == 0) |
| return bestRule; |
| |
| sendStatusUpdateToDelegate( |
| "Round " |
| + roundNumber |
| + "." |
| + subRoundNumber |
| + " - Testing " |
| + rulesToTest.size() |
| + " rules... " |
| + " - uncovered examples: " |
| + (allExamplesCount - coveredExamples.size() + " / " + allExamplesCount |
| + " ; cs=" + cachedTestedRuleStatistics.size()), |
| TextRulerLearnerState.ML_RUNNING, false); |
| |
| TextRulerToolkit.log("Testing " + rulesToTest.size() + " rules on training set..."); |
| for (TextRulerRule r : rulesToTest) |
| TextRulerToolkit.log(r.getRuleString()); |
| testRulesIfNotCached(rulesToTest); |
| |
| if (shouldAbort()) |
| return null; |
| for (TextRulerRule r : rulesToTest) { |
| WhiskRule wr = (WhiskRule) r; |
| if (wr.getLaplacian() < bestL) { |
| bestL = wr.getLaplacian(); |
| bestRule = wr; |
| bestRuleConstraintPoints = bestRule.totalConstraintPoints(); |
| } else if (wr.getLaplacian() == bestL && bestRuleConstraintPoints >= 0) { |
| TextRulerToolkit.log("Same Laplacian! So prefer more general rule!"); |
| if (wr.totalConstraintPoints() < bestRuleConstraintPoints) { |
| TextRulerToolkit.log("\tYes, prefered!"); |
| bestL = wr.getLaplacian(); |
| bestRule = wr; |
| bestRuleConstraintPoints = bestRule.totalConstraintPoints(); |
| } |
| } |
| } |
| return bestRule; |
| } |
| |
| private List<List<WhiskRuleItem>> getTermsWithinWindow(List<List<WhiskRuleItem>> slotTerms, |
| TextRulerExample example, int steps) { |
| if (steps == windowSize) |
| return slotTerms; |
| List<List<WhiskRuleItem>> result = new ArrayList<List<WhiskRuleItem>>(); |
| |
| for (List<WhiskRuleItem> list : slotTerms) { |
| List<WhiskRuleItem> termsBefore = getTermsBefore(list.get(0), example); |
| List<WhiskRuleItem> termsAfter = getTermsAfter(list.get(list.size() - 1), example); |
| if (!termsBefore.isEmpty()) { |
| for (WhiskRuleItem before : termsBefore) { |
| for (WhiskRuleItem after : termsAfter) { |
| List<WhiskRuleItem> newList = new ArrayList<WhiskRuleItem>(); |
| newList.add(before); |
| newList.addAll(list); |
| newList.add(after); |
| result.add(newList); |
| } |
| } |
| } else { |
| for (WhiskRuleItem after : termsAfter) { |
| List<WhiskRuleItem> newList = new ArrayList<WhiskRuleItem>(); |
| newList.addAll(list); |
| newList.add(after); |
| result.add(newList); |
| } |
| } |
| } |
| result = getTermsWithinWindow(result, example, ++steps); |
| return result; |
| } |
| |
| protected WhiskRule createNewRuleByAddingTerm(WhiskRule baseRule, WhiskRuleItem term) { |
| if (term == null) |
| return null; |
| if (term.isStarWildCard() || term.getWordConstraint() == null) |
| return null; |
| WhiskRule newRule = baseRule.copy(); |
| // int foundSlotNumber = -1; // debug info |
| // String foundSlotPattern = ""; |
| int termBeginNumber = term.getWordConstraint().getTokenAnnotation().getBegin(); |
| int termEndNumber = term.getWordConstraint().getTokenAnnotation().getEnd(); |
| TextRulerRulePattern targetPattern = null; |
| TextRulerRulePattern previousSlotPostFillerPattern = null; |
| for (int i = 0; i < newRule.getPatterns().size(); i++) { |
| TextRulerSlotPattern slotPattern = newRule.getPatterns().get(i); |
| WhiskRuleItem it = (WhiskRuleItem) slotPattern.preFillerPattern.lastItem(); // look at the |
| // prefiller |
| // pattern |
| if (it != null && it.getWordConstraint() != null |
| && termEndNumber <= it.getWordConstraint().getTokenAnnotation().getBegin()) |
| targetPattern = slotPattern.preFillerPattern; |
| if (targetPattern == null && slotPattern.fillerPattern.size() > 0) // now |
| // look |
| // at |
| // the |
| // filler |
| // pattern |
| { |
| it = (WhiskRuleItem) slotPattern.fillerPattern.firstItem(); |
| if (it.getWordConstraint() != null |
| && termEndNumber <= it.getWordConstraint().getTokenAnnotation().getBegin()) // it's |
| // still |
| // for |
| // the prefiller |
| // pattern but it |
| // seems to be |
| // emtpy so we |
| // could not find |
| // that out above! |
| targetPattern = slotPattern.preFillerPattern; |
| else { |
| it = (WhiskRuleItem) slotPattern.fillerPattern.lastItem(); |
| if (it.getWordConstraint() != null |
| && termEndNumber <= it.getWordConstraint().getTokenAnnotation().getBegin()) { |
| targetPattern = slotPattern.fillerPattern; |
| } |
| } |
| } |
| if (targetPattern == null && slotPattern.postFillerPattern.size() > 0) // now |
| // look |
| // at |
| // the |
| // postfiller |
| // pattern |
| { |
| it = (WhiskRuleItem) slotPattern.postFillerPattern.firstItem(); |
| if (it.getWordConstraint() != null |
| && termEndNumber <= it.getWordConstraint().getTokenAnnotation().getBegin()) // it's |
| // still |
| // for |
| // the filler |
| // pattern but it |
| // seems to be |
| // emtpy so we |
| // could not find |
| // that out above! |
| targetPattern = slotPattern.fillerPattern; |
| else { |
| it = (WhiskRuleItem) slotPattern.postFillerPattern.lastItem(); |
| if (it.getWordConstraint() != null |
| && termEndNumber <= it.getWordConstraint().getTokenAnnotation().getBegin()) |
| targetPattern = slotPattern.postFillerPattern; |
| } |
| } |
| if (targetPattern == null) { |
| targetPattern = previousSlotPostFillerPattern; |
| // debug info |
| // if (i > 0) { |
| // TextRulerSlotPattern prevSlotPattern = newRule.getPatterns().get(i - |
| // 1); |
| // foundSlotPattern = targetPattern == prevSlotPattern.preFillerPattern |
| // ? "PRE FILLER" |
| // : (targetPattern == prevSlotPattern.fillerPattern ? "FILLER" : |
| // "POST FILLER"); |
| // foundSlotNumber = i - 1; |
| // } |
| // } else { |
| // foundSlotPattern = targetPattern == slotPattern.preFillerPattern ? |
| // "PRE FILLER" |
| // : (targetPattern == slotPattern.fillerPattern ? "FILLER" : |
| // "POST FILLER"); |
| // foundSlotNumber = i; |
| } |
| previousSlotPostFillerPattern = slotPattern.postFillerPattern; |
| } |
| |
| if (targetPattern == null) { |
| targetPattern = previousSlotPostFillerPattern; |
| // debug info |
| // foundSlotNumber = newRule.getPatterns().size() - 1; |
| // foundSlotPattern = "POST FILLER"; |
| } |
| |
| if (targetPattern == null) { |
| TextRulerToolkit.log("ERROR, NO TARGET PATTERN FOR NEW RULE TERM FOUND !"); |
| } else { |
| // TextRulerToolkit.log("Ok, found for Rule: "+newRule.getRuleString()); |
| // TextRulerToolkit.log("Term: "+term.getTermNumberInExample()+" ; "+term); |
| // TextRulerToolkit.log("Slot "+foundSlotNumber+" - Pattern: "+foundSlotPattern); |
| // now put that term into the rule: |
| int indexInPattern = -1; |
| if (targetPattern.size() == 0) { |
| targetPattern.add(term.copy()); |
| indexInPattern = 0; |
| } else { |
| // 1. search if the term would replace a wildcard: |
| WhiskRuleItem wildCard = null; |
| for (TextRulerRuleItem i : newRule.getPatterns().get(0).preFillerPattern) { |
| if (((WhiskRuleItem) i).isStarWildCard()) { |
| WhiskRuleItem left = newRule.searchNeighborOfItem(((WhiskRuleItem) i), true); |
| WhiskRuleItem right = newRule.searchNeighborOfItem(((WhiskRuleItem) i), false); |
| if (left.getWordConstraint().getTokenAnnotation().getEnd() <= termBeginNumber |
| && right.getWordConstraint().getTokenAnnotation().getBegin() >= termEndNumber) |
| wildCard = (WhiskRuleItem) i; |
| } |
| } |
| if (wildCard == null) { |
| for (TextRulerRuleItem i : newRule.getPatterns().get(0).fillerPattern) { |
| if (((WhiskRuleItem) i).isStarWildCard()) { |
| WhiskRuleItem left = newRule.searchNeighborOfItem(((WhiskRuleItem) i), true); |
| WhiskRuleItem right = newRule.searchNeighborOfItem(((WhiskRuleItem) i), false); |
| if (left != null |
| && left.getWordConstraint().getTokenAnnotation().getEnd() <= termBeginNumber |
| && right.getWordConstraint().getTokenAnnotation().getBegin() >= termEndNumber) |
| wildCard = (WhiskRuleItem) i; |
| } |
| } |
| } |
| if (wildCard == null) { |
| for (TextRulerRuleItem i : newRule.getPatterns().get(0).postFillerPattern) { |
| if (((WhiskRuleItem) i).isStarWildCard()) { |
| WhiskRuleItem left = newRule.searchNeighborOfItem(((WhiskRuleItem) i), true); |
| WhiskRuleItem right = newRule.searchNeighborOfItem(((WhiskRuleItem) i), false); |
| if (left.getWordConstraint().getTokenAnnotation().getEnd() <= termBeginNumber |
| && right.getWordConstraint().getTokenAnnotation().getBegin() >= termEndNumber) |
| wildCard = (WhiskRuleItem) i; |
| } |
| } |
| } |
| if (wildCard != null) { |
| if (!wildCard.isStarWildCard()) { |
| TextRulerToolkit |
| .log("ERROR, FOUND A TERM WITH THE SAME NUMBER THAT IS NOT A WILDCARD! HOW IS THAT???"); |
| return null; |
| } |
| if (!targetPattern.contains(wildCard)) { |
| TextRulerToolkit.log("EVEN WORSE, THAT MUST NOT BE AT ALL!"); |
| return null; |
| } |
| indexInPattern = targetPattern.indexOf(wildCard); |
| targetPattern.set(indexInPattern, term.copy()); |
| } else { |
| // not a wildcard, so search for the insertion point: |
| for (int i = 0; i < targetPattern.size(); i++) { |
| WhiskRuleItem it = (WhiskRuleItem) targetPattern.get(i); |
| if (it.getWordConstraint() != null |
| && termEndNumber <= it.getWordConstraint().getTokenAnnotation().getBegin()) { |
| indexInPattern = i; |
| break; |
| } |
| } |
| if (indexInPattern < 0) { |
| indexInPattern = targetPattern.size(); |
| targetPattern.add(term.copy()); |
| } else |
| targetPattern.add(indexInPattern, term.copy()); |
| } |
| } |
| // ok, now we have replaced a wildcard with the term or added the |
| // term between two other items. |
| // we now have to check the neighbors of the new term: if it is a |
| // direct neighbor (according to the termNumber), |
| // we have nothing special to do. but if it is not a direct |
| // neighbor, we have to add a wildcard between the two items (if the |
| // neighbor item |
| // is not a wildcard itself! |
| WhiskRuleItem newTerm = (WhiskRuleItem) targetPattern.get(indexInPattern); |
| |
| // look at left neighbor: |
| WhiskRuleItem left = newRule.searchNeighborOfItem(newTerm, true); |
| if (left != null && left.getWordConstraint() != null) { |
| // TextRulerToolkit.log("LEFT NEIGHBOR FOUND!"); |
| |
| // so we have a left neighbor. let's see if it also is the |
| // neighbor in our seed token stream: |
| if (!left.isStarWildCard()) { // no direct neighbor and |
| // no wildcard yet, |
| // so insert a wildcard between us! |
| boolean isValid = isNextValidNeighbor(left, newTerm, newRule.getSeedExample()); |
| if (!isValid) { |
| targetPattern.add(indexInPattern, WhiskRuleItem.newWildCardItem()); |
| indexInPattern++; |
| } |
| } |
| } |
| |
| // look at right neighbor: |
| WhiskRuleItem right = newRule.searchNeighborOfItem(newTerm, false); |
| if (right != null && right.getWordConstraint() != null) { |
| // TextRulerToolkit.log("RIGHT NEIGHBOR FOUND!"); |
| // so we have a right neighbor. let's see if it also is the |
| // neighbor in our seed token stream: |
| if (!right.isStarWildCard()) { |
| // no direct neighbor and |
| // no wildcard yet, |
| // so insert a wildcard between us! |
| boolean isValid = isNextValidNeighbor(newTerm, right, newRule.getSeedExample()); |
| if (!isValid) { |
| WhiskRuleItem wc = WhiskRuleItem.newWildCardItem(); |
| if (indexInPattern + 1 < targetPattern.size()) |
| targetPattern.add(indexInPattern + 1, wc); |
| else |
| targetPattern.add(wc); |
| } |
| } |
| } |
| |
| newRule.setNeedsCompile(true); |
| // TextRulerToolkit.log("BEFORE: "+baseRule.getRuleString()); |
| // TextRulerToolkit.log("AFTER : "+newRule.getRuleString()); |
| // TextRulerToolkit.log(""); |
| } |
| if (newRule.getRuleString().equals(baseRule.getRuleString())) // this |
| // must |
| // not be! |
| return null; |
| else |
| return newRule; |
| } |
| |
| protected WhiskRule anchor(WhiskRule rule, TextRulerExampleDocument doc, |
| TextRulerExample example, int slotIndex) { |
| List<WhiskRule> result = new ArrayList<WhiskRule>(); |
| TextRulerAnnotation slotAnnotation = example.getAnnotations()[slotIndex]; |
| List<List<WhiskRuleItem>> window = getTermsWithinBounds(slotAnnotation.getBegin(), |
| slotAnnotation.getEnd(), example); |
| |
| for (List<WhiskRuleItem> inside : window) { |
| |
| if (rule == null || inside.isEmpty()) { |
| return null; |
| } |
| // create base 1 and base 2: |
| WhiskRule base1 = rule.copy(); // slot filler rule |
| TextRulerSlotPattern slotPattern = base1.getPatterns().get(slotIndex); |
| // questionable restriction: |
| if (inside.size() <= windowSize) { // TODO add parameter for this! |
| slotPattern.fillerPattern.addAll(inside); |
| } else { |
| for (int i = 0; i < inside.size(); i++) |
| if (i == 0 || (i == inside.size() - 1)) |
| slotPattern.fillerPattern.add(inside.get(i).copy()); |
| else if (inside.size() > 2 && i < 2) |
| slotPattern.fillerPattern.add(WhiskRuleItem.newWildCardItem()); |
| } |
| List<WhiskRuleItem> beforeList = getTermsBefore(inside.get(0), example); |
| List<WhiskRuleItem> afterList = getTermsAfter(inside.get(inside.size() - 1), example); |
| beforeList.add(null); |
| afterList.add(null); |
| Collection<WhiskRule> tempRules = new HashSet<WhiskRule>(); |
| |
| // workaround for better rules: |
| // only inner begin |
| for (WhiskRuleItem eachBefore : beforeList) { |
| for (WhiskRuleItem eachAfter : afterList) { |
| WhiskRule copy = rule.copy(); |
| TextRulerSlotPattern textRulerSlotPattern = copy.getPatterns().get(slotIndex); |
| if (eachBefore != null) { |
| textRulerSlotPattern.preFillerPattern.add(eachBefore); |
| } |
| textRulerSlotPattern.fillerPattern.add(inside.get(0).copy()); |
| textRulerSlotPattern.fillerPattern.add(WhiskRuleItem.newWildCardItem()); |
| if (eachAfter != null) { |
| textRulerSlotPattern.postFillerPattern.add(eachAfter); |
| } |
| tempRules.add(copy); |
| } |
| } |
| // only inner end |
| for (WhiskRuleItem eachBefore : beforeList) { |
| for (WhiskRuleItem eachAfter : afterList) { |
| WhiskRule copy = rule.copy(); |
| TextRulerSlotPattern textRulerSlotPattern = copy.getPatterns().get(slotIndex); |
| if (eachBefore != null) { |
| textRulerSlotPattern.preFillerPattern.add(eachBefore); |
| } |
| textRulerSlotPattern.fillerPattern.add(WhiskRuleItem.newWildCardItem()); |
| textRulerSlotPattern.fillerPattern.add(inside.get(inside.size() - 1).copy()); |
| if (eachAfter != null) { |
| textRulerSlotPattern.postFillerPattern.add(eachAfter); |
| } |
| tempRules.add(copy); |
| } |
| } |
| |
| if (!beforeList.isEmpty()) { |
| if (!afterList.isEmpty()) { |
| for (WhiskRuleItem eachBefore : beforeList) { |
| for (WhiskRuleItem eachAfter : afterList) { |
| WhiskRule copy = rule.copy(); |
| TextRulerSlotPattern textRulerSlotPattern = copy.getPatterns().get(slotIndex); |
| if (eachBefore != null) { |
| textRulerSlotPattern.preFillerPattern.add(eachBefore); |
| } |
| textRulerSlotPattern.fillerPattern.add(WhiskRuleItem.newWildCardItem()); |
| if (eachAfter != null) { |
| textRulerSlotPattern.postFillerPattern.add(eachAfter); |
| } |
| tempRules.add(copy); |
| } |
| } |
| } else { |
| for (WhiskRuleItem eachBefore : beforeList) { |
| WhiskRule copy = rule.copy(); |
| TextRulerSlotPattern textRulerSlotPattern = copy.getPatterns().get(slotIndex); |
| textRulerSlotPattern.fillerPattern.add(WhiskRuleItem.newWildCardItem()); |
| if (eachBefore != null) { |
| textRulerSlotPattern.preFillerPattern.add(eachBefore); |
| } |
| tempRules.add(copy); |
| } |
| } |
| } else { |
| for (WhiskRuleItem eachAfter : afterList) { |
| WhiskRule copy = rule.copy(); |
| TextRulerSlotPattern textRulerSlotPattern = copy.getPatterns().get(slotIndex); |
| textRulerSlotPattern.fillerPattern.add(WhiskRuleItem.newWildCardItem()); |
| if (eachAfter != null) { |
| textRulerSlotPattern.postFillerPattern.add(eachAfter); |
| } |
| tempRules.add(copy); |
| } |
| } |
| ArrayList<TextRulerRule> rules = new ArrayList<TextRulerRule>(tempRules); |
| testRulesIfNotCached(rules); |
| TextRulerRule best = null; |
| for (TextRulerRule each : rules) { |
| if (best == null) { |
| best = each; |
| } else { |
| if (each.getCoveringStatistics().getCoveredPositivesCount() > best |
| .getCoveringStatistics().getCoveredPositivesCount()) { |
| best = each; |
| } |
| } |
| } |
| WhiskRule base2 = (WhiskRule) best; |
| List<TextRulerRule> testRules = new ArrayList<TextRulerRule>(); |
| if (base1 != null) { |
| TextRulerToolkit.log("base1: " + base1.getRuleString()); |
| testRules.add(base1); |
| } |
| if (base2 != null) { |
| TextRulerToolkit.log("base2: " + base2.getRuleString()); |
| testRules.add(base2); |
| } |
| testRulesIfNotCached(testRules); |
| if (shouldAbort()) { |
| return null; |
| } |
| if (base1 != null && base2 == null) { |
| TextRulerToolkit.log("\tbase1: " + base1.getCoveringStatistics() + " --> laplacian = " |
| + base1.getLaplacian()); |
| result.add(base1); |
| } else { |
| TextRulerToolkit.log("\tbase1: " + base1.getCoveringStatistics() + " --> laplacian = " |
| + base1.getLaplacian()); |
| TextRulerToolkit.log("\tbase2: " + base2.getCoveringStatistics() + " --> laplacian = " |
| + base2.getLaplacian()); |
| if (base2.getCoveringStatistics().getCoveredPositivesCount() > base1 |
| .getCoveringStatistics().getCoveredPositivesCount()) { |
| result.add(base2); |
| } else { |
| result.add(base1); |
| } |
| } |
| } |
| TextRulerRule best = null; |
| for (TextRulerRule each : result) { |
| if (best == null) { |
| best = each; |
| } else { |
| if (each.getCoveringStatistics().getCoveredPositivesCount() > best.getCoveringStatistics() |
| .getCoveredPositivesCount()) { |
| best = each; |
| } |
| } |
| } |
| |
| return (WhiskRule) best; |
| } |
| |
| private List<WhiskRuleItem> getTermsAfter(WhiskRuleItem whiskRuleItem, TextRulerExample example) { |
| List<WhiskRuleItem> result = new ArrayList<WhiskRuleItem>(); |
| int end = whiskRuleItem.getWordConstraint().getTokenAnnotation().getEnd(); |
| CAS cas = example.getDocumentCAS(); |
| // TODO: access type with string constant |
| Type frameType = cas.getTypeSystem().getType("org.apache.uima.ruta.type.RutaFrame"); |
| AnnotationFS pointer = cas.createAnnotation(frameType, end, Integer.MAX_VALUE); |
| FSIterator iterator = cas.getAnnotationIndex().iterator(pointer); |
| int nextBegin = -1; |
| while (iterator.isValid()) { |
| FeatureStructure fs = iterator.get(); |
| if (fs instanceof AnnotationFS) { |
| AnnotationFS a = (AnnotationFS) fs; |
| if (!filterSetWithSlotNames.contains(a.getType().getName())) { |
| if (nextBegin == -1) { |
| nextBegin = a.getBegin(); |
| } else if (nextBegin != a.getBegin()) { |
| break; |
| } |
| if (a.getBegin() <= nextBegin && a.getBegin() >= end) { |
| WhiskRuleItem term = new WhiskRuleItem(new TextRulerAnnotation(a, |
| example.getDocument(), consideredFeatures)); |
| result.add(term); |
| } |
| } |
| } |
| iterator.moveToNext(); |
| } |
| return result; |
| } |
| |
| private List<WhiskRuleItem> getTermsBefore(WhiskRuleItem whiskRuleItem, TextRulerExample example) { |
| List<WhiskRuleItem> result = new ArrayList<WhiskRuleItem>(); |
| int begin = whiskRuleItem.getWordConstraint().getTokenAnnotation().getBegin(); |
| CAS cas = example.getDocumentCAS(); |
| |
| // TODO: access type with string constant |
| Type frameType = cas.getTypeSystem().getType("org.apache.uima.ruta.type.RutaFrame"); |
| AnnotationFS pointer = cas.createAnnotation(frameType, begin, begin); |
| FSIterator iterator = cas.getAnnotationIndex().iterator(pointer); |
| int nextEnd = -1; |
| |
| // ??? |
| iterator.moveToPrevious(); |
| iterator.moveToPrevious(); |
| while (iterator.isValid()) { |
| FeatureStructure fs = iterator.get(); |
| if (fs instanceof AnnotationFS) { |
| AnnotationFS a = (AnnotationFS) fs; |
| if (!filterSetWithSlotNames.contains(a.getType().getName())) { |
| if (a.getEnd() > example.getAnnotation().getEnd()) { |
| iterator.moveToPrevious(); |
| continue; |
| } |
| if (nextEnd == -1) { |
| nextEnd = a.getEnd(); |
| } else if (nextEnd != a.getEnd()) { |
| break; |
| } |
| if (a.getEnd() >= nextEnd && a.getEnd() <= begin) { |
| WhiskRuleItem term = new WhiskRuleItem(new TextRulerAnnotation(a, |
| example.getDocument(), consideredFeatures)); |
| result.add(term); |
| } |
| } |
| } |
| iterator.moveToPrevious(); |
| } |
| return result; |
| } |
| |
| public String getResultString() { |
| if (ruleList != null) |
| return getFileHeaderString(true) + ruleList.getRulesString(""); |
| else |
| return "No results available yet!"; |
| } |
| |
| public void setParameters(Map<String, Object> params) { |
| if (TextRulerToolkit.DEBUG) |
| saveParametersToTempFolder(params); |
| |
| // TODO try catch |
| if (params.containsKey(WINDOWSIZE_KEY)) |
| windowSize = (Integer) params.get(WINDOWSIZE_KEY); |
| |
| if (params.containsKey(ERROR_THRESHOLD_KEY)) |
| errorThreshold = (Float) params.get(ERROR_THRESHOLD_KEY); |
| |
| if (params.containsKey(POSTAG_ROOTTYPE_KEY)) |
| posTagRootTypeName = (String) params.get(POSTAG_ROOTTYPE_KEY); |
| |
| if (params.containsKey(CONSIDERED_FEATURES)) { |
| String list = (String) params.get(CONSIDERED_FEATURES); |
| if (!StringUtils.isBlank(list)) { |
| String[] split = list.split(","); |
| for (String string : split) { |
| String trim = string.trim(); |
| if (!StringUtils.isBlank(trim)) { |
| consideredFeatures.add(trim); |
| } |
| } |
| } |
| } |
| |
| } |
| |
| public List<List<WhiskRuleItem>> getTermsWithinBounds(int startPos, int endPos, |
| TextRulerExample example) { |
| List<List<WhiskRuleItem>> result = new ArrayList<List<WhiskRuleItem>>(); |
| CAS cas = example.getDocumentCAS(); |
| // TODO: access type with string constant |
| Type frameType = cas.getTypeSystem().getType("org.apache.uima.ruta.type.RutaFrame"); |
| AnnotationFS pointer = cas.createAnnotation(frameType, startPos, endPos); |
| FSIterator iterator = cas.getAnnotationIndex().iterator(pointer); |
| List<AnnotationFS> startAs = new ArrayList<AnnotationFS>(); |
| int firstBegin = -1; |
| while (iterator.isValid()) { |
| FeatureStructure fs = iterator.get(); |
| AnnotationFS a = (AnnotationFS) fs; |
| |
| // TODO change for multislot rules! |
| if (a.getBegin() >= startPos && a.getEnd() <= endPos) { |
| if (!filterSetWithSlotNames.contains(a.getType().getName())) { |
| if (firstBegin == -1) { |
| firstBegin = a.getBegin(); |
| } else if (firstBegin != a.getBegin()) { |
| break; |
| } |
| if (a.getBegin() == firstBegin) |
| startAs.add(a); |
| } |
| iterator.moveToNext(); |
| } else { |
| iterator.moveToNext(); |
| } |
| } |
| |
| for (AnnotationFS annotation : startAs) { |
| List<WhiskRuleItem> startList = new ArrayList<WhiskRuleItem>(); |
| WhiskRuleItem term = new WhiskRuleItem(new TextRulerAnnotation(annotation, |
| example.getDocument(), consideredFeatures)); |
| startList.add(term); |
| result.add(startList); |
| } |
| |
| result = addFollowing(result, endPos, example); |
| return result; |
| } |
| |
| private List<List<WhiskRuleItem>> addFollowing(List<List<WhiskRuleItem>> lists, int till, |
| TextRulerExample example) { |
| List<List<WhiskRuleItem>> result = new ArrayList<List<WhiskRuleItem>>(); |
| for (List<WhiskRuleItem> list : lists) { |
| WhiskRuleItem last = list.get(list.size() - 1); |
| List<WhiskRuleItem> termsAfter = getTermsAfter(last, example); |
| if (termsAfter.isEmpty()) { |
| return lists; |
| } |
| for (WhiskRuleItem eachAfter : termsAfter) { |
| if (eachAfter.getWordConstraint().getTokenAnnotation().getEnd() <= till) { |
| List<WhiskRuleItem> newList = new ArrayList<WhiskRuleItem>(); |
| newList.addAll(list); |
| newList.add(eachAfter); |
| result.add(newList); |
| result = addFollowing(result, till, example); |
| } else { |
| return lists; |
| } |
| } |
| } |
| return result; |
| } |
| |
| // TODO share this between algorithms (e.g. LP2 and RAPIER, WHISK ?) and |
| // make a maximum size of the cache, etc. like CasCache? |
| protected void testRulesIfNotCached(List<TextRulerRule> rules) { |
| List<TextRulerRule> rulesToTest = new ArrayList<TextRulerRule>(); |
| |
| for (TextRulerRule r : rules) { |
| String key = r.getRuleString(); |
| if (cachedTestedRuleStatistics.containsKey(key)) { |
| r.setCoveringStatistics(cachedTestedRuleStatistics.get(key).copy()); |
| } else |
| rulesToTest.add(r); |
| } |
| |
| if (rulesToTest.size() > 0) { |
| testRulesOnDocumentSet(rulesToTest, exampleDocuments); |
| if (shouldAbort()) |
| return; |
| for (TextRulerRule r : rulesToTest) { |
| String key = r.getRuleString(); |
| cachedTestedRuleStatistics.put(key, r.getCoveringStatistics().copy()); |
| } |
| } |
| } |
| |
| private int getElementIndex(WhiskRule proposedRule, WhiskRuleItem term) { |
| if (term == null) |
| return -1; |
| int index = 0; |
| int result = -1; |
| for (TextRulerRuleItem i : proposedRule.getPatterns().get(0).preFillerPattern) { |
| if (((WhiskRuleItem) i).equals(term)) { |
| result = index; |
| } |
| index++; |
| } |
| for (TextRulerRuleItem i : proposedRule.getPatterns().get(0).fillerPattern) { |
| if (((WhiskRuleItem) i).equals(term)) { |
| result = index; |
| } |
| index++; |
| } |
| for (TextRulerRuleItem i : proposedRule.getPatterns().get(0).postFillerPattern) { |
| if (((WhiskRuleItem) i).equals(term)) { |
| result = index; |
| } |
| index++; |
| } |
| return result; |
| } |
| |
| private boolean isNextValidNeighbor(WhiskRuleItem left, WhiskRuleItem right, |
| TextRulerExample example) { |
| CAS cas = example.getDocumentCAS(); |
| // TODO: access type with string constant |
| Type frameType = cas.getTypeSystem().getType("org.apache.uima.ruta.type.RutaFrame"); |
| int begin = left.getWordConstraint().getTokenAnnotation().getEnd(); |
| int end = right.getWordConstraint().getTokenAnnotation().getBegin(); |
| AnnotationFS pointer = cas.createAnnotation(frameType, begin, end); |
| FSIterator iterator = cas.getAnnotationIndex().iterator(pointer); |
| while (iterator.isValid()) { |
| FeatureStructure fs = iterator.get(); |
| AnnotationFS a = (AnnotationFS) fs; |
| if (a.getBegin() >= begin && a.getEnd() <= end) { |
| if (!filterSetWithSlotNames.contains(a.getType().getName())) { |
| return false; |
| } |
| } |
| iterator.moveToNext(); |
| } |
| return true; |
| } |
| } |
