| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| |
| package org.apache.uima.ruta.textruler.learner.rapier; |
| |
| import java.util.ArrayList; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Random; |
| import java.util.Set; |
| |
| import org.apache.uima.cas.CAS; |
| import org.apache.uima.cas.Type; |
| import org.apache.uima.cas.TypeSystem; |
| import org.apache.uima.cas.text.AnnotationFS; |
| import org.apache.uima.ruta.textruler.core.TextRulerAnnotation; |
| import org.apache.uima.ruta.textruler.core.TextRulerBasicLearner; |
| import org.apache.uima.ruta.textruler.core.TextRulerExample; |
| import org.apache.uima.ruta.textruler.core.TextRulerRule; |
| import org.apache.uima.ruta.textruler.core.TextRulerRuleItem; |
| import org.apache.uima.ruta.textruler.core.TextRulerRuleList; |
| import org.apache.uima.ruta.textruler.core.TextRulerRulePattern; |
| import org.apache.uima.ruta.textruler.core.TextRulerStatisticsCollector; |
| import org.apache.uima.ruta.textruler.core.TextRulerTarget; |
| import org.apache.uima.ruta.textruler.core.TextRulerToolkit; |
| import org.apache.uima.ruta.textruler.core.TextRulerWordConstraint; |
| import org.apache.uima.ruta.textruler.extension.TextRulerLearnerDelegate; |
| |
| public class Rapier extends TextRulerBasicLearner { |
| |
| public final static String COMPRESSION_FAIL_MAX_COUNT_KEY = "compressionFailMaxCount"; |
| |
| public final static String RULELIST_SIZE_KEY = "ruleListSize"; |
| |
| public final static String PAIR_COUNT_KEY = "pairCount"; |
| |
| public final static String LIM_NO_IMPROVEMENTS_KEY = "limNoImprovements"; |
| |
| public final static String NOISE_THESHOLD_KEY = "noiseThreshold"; |
| |
| public final static String POSTAG_ROOTTYPE_KEY = "posTagRootType"; |
| |
| public final static String MIN_COVERED_POSITIVES_KEY = "minCoveredPositives"; |
| |
| public final static String USE_ALL_GENSETS_AT_SPECIALIZATION_KEY = "useAllGenSetsAtSpecialization"; |
| |
| public final static int STANDARD_COMPRESSION_FAIL_MAX_COUNT = 3; |
| |
| public final static int STANDARD_RULELIST_SIZE = 50; |
| |
| public final static int STANDARD_PAIR_COUNT = 4; |
| |
| public final static int STANDARD_LIM_NO_IMPROVEMENTS = 3; |
| |
| public final static float STANDARD_NOISE_THREHSOLD = 0.9f; |
| |
| public final static String STANDARD_POSTAG_ROOTTYPE = "org.apache.uima.ml.ML.postag"; |
| |
| public final static int STANDARD_MIN_COVERED_POSITIVES = 1; |
| |
| public final static boolean STANDARD_USE_ALL_GENSETS_AT_SPECIALIZATION = true; |
| |
| private int compressionFailMaxCount = STANDARD_COMPRESSION_FAIL_MAX_COUNT; |
| |
| private int ruleListSize = STANDARD_RULELIST_SIZE; |
| |
| private int pairCount = STANDARD_PAIR_COUNT; |
| |
| private int limNoImprovements = STANDARD_LIM_NO_IMPROVEMENTS; |
| |
| private float noiseThreshold = STANDARD_NOISE_THREHSOLD; |
| |
| private String posTagRootTypeName = STANDARD_POSTAG_ROOTTYPE; |
| |
| private int minCoveredPositives = STANDARD_MIN_COVERED_POSITIVES; |
| |
| private boolean useAllGenSetsAtSpecialization = STANDARD_USE_ALL_GENSETS_AT_SPECIALIZATION; |
| |
| private Map<String, TextRulerStatisticsCollector> cachedTestedRuleStatistics = new HashMap<String, TextRulerStatisticsCollector>(); |
| |
| private int initialRuleBaseSize; |
| |
| private List<TextRulerExample> examples; |
| |
| private TextRulerRuleList slotRules; |
| |
| private RapierRulePriorityQueue ruleList; |
| |
| private String currentSlotName; |
| |
| public Rapier(String inputDir, String prePropTMFile, String tmpDir, String[] slotNames, |
| Set<String> filterSet, boolean skip, TextRulerLearnerDelegate delegate) { |
| super(inputDir, prePropTMFile, tmpDir, slotNames, filterSet, skip, delegate); |
| } |
| |
| @Override |
| protected void doRun() { |
| for (int i = 0; i < slotNames.length; i++) { |
| int compressionFailCount = 0; |
| |
| // only working for one slot yet ! |
| currentSlotName = slotNames[i]; |
| cachedTestedRuleStatistics.clear(); |
| exampleDocuments.createExamplesForTarget(new TextRulerTarget(currentSlotName, this)); |
| examples = exampleDocuments.getAllPositiveExamples(); |
| |
| if (shouldAbort()) |
| return; |
| |
| slotRules = new TextRulerRuleList(); |
| ruleList = new RapierRulePriorityQueue(ruleListSize); |
| |
| TextRulerToolkit.log("--- RAPIER START for Slot " + currentSlotName); |
| |
| sendStatusUpdateToDelegate("Creating initial rule base...", |
| TextRulerLearnerState.ML_INITIALIZING, false); |
| |
| fillSlotRulesWithMostSpecificRules(); |
| |
| updateCompressionStatusString(); |
| |
| if (TextRulerToolkit.DEBUG) { |
| slotRules.saveToRulesFile(getIntermediateRulesFileName(), getFileHeaderString(true)); |
| } |
| |
| while (compressionFailCount < compressionFailMaxCount) { |
| TextRulerToolkit.log("***** NEW COMPRESSION ROUND; FailCount = " + compressionFailCount); |
| if (shouldAbort()) { |
| return; |
| } |
| |
| RapierRule bestRule = findNewRule(); |
| if (bestRule != null |
| && (bestRule.getCoveringStatistics().getCoveredPositivesCount() >= minCoveredPositives) |
| && (bestRule.noiseValue() >= noiseThreshold) && (!slotRules.contains(bestRule))) { |
| addRuleAndRemoveEmpiricallySubsumedRules(bestRule); |
| if (TextRulerToolkit.DEBUG) |
| slotRules.saveToRulesFile(getIntermediateRulesFileName(), getFileHeaderString(true)); |
| } else { |
| compressionFailCount++; |
| } |
| } |
| |
| if (TextRulerToolkit.DEBUG) { |
| slotRules.saveToRulesFile(getIntermediateRulesFileName(), getFileHeaderString(true)); |
| } |
| } |
| |
| sendStatusUpdateToDelegate("Done", TextRulerLearnerState.ML_DONE, true); |
| cachedTestedRuleStatistics.clear(); |
| TextRulerToolkit.log("--- RAPIER END"); |
| |
| } |
| |
| private void updateCompressionStatusString() { |
| double percent = Math.round((slotRules.size() / (double) initialRuleBaseSize) * 100.0); |
| sendStatusUpdateToDelegate("Compressing... (Rules = " + slotRules.size() + "/" |
| + initialRuleBaseSize + " = " + percent + " % ratio)", |
| TextRulerLearnerState.ML_RUNNING, true); |
| // TODO also show round numbers and compression fail count and such |
| // things! |
| } |
| |
| private void addAvailablePosTagConstraintToItem(RapierRuleItem item, |
| AnnotationFS tokenAnnotation, TextRulerExample example) { |
| |
| if (posTagRootTypeName != null && posTagRootTypeName.length() > 0) { |
| CAS cas = example.getDocumentCAS(); |
| TypeSystem ts = cas.getTypeSystem(); |
| Type posTagsRootType = ts.getType(posTagRootTypeName); |
| if (ts != null) { |
| List<AnnotationFS> posTagAnnotations = TextRulerToolkit.getAnnotationsWithinBounds(cas, |
| tokenAnnotation.getBegin(), tokenAnnotation.getEnd(), null, posTagsRootType); |
| if (posTagAnnotations.size() > 0) { |
| AnnotationFS posTag = posTagAnnotations.get(0); |
| if (posTag.getBegin() == tokenAnnotation.getBegin() |
| && posTag.getEnd() == tokenAnnotation.getEnd()) |
| item.addTagConstraint(posTag.getType().getShortName()); |
| } |
| } |
| } |
| } |
| |
| private void fillSlotRulesWithMostSpecificRules() { |
| slotRules.clear(); |
| for (TextRulerExample example : examples) { |
| RapierRule rule = new RapierRule(this, example.getTarget()); |
| TextRulerAnnotation slotAnnotation = example.getAnnotation(); |
| CAS docCas = example.getDocumentCAS(); |
| TypeSystem ts = docCas.getTypeSystem(); |
| Type tokensRootType = ts.getType(TextRulerToolkit.RUTA_ANY_TYPE_NAME); |
| |
| // first, get all words/tokens: |
| List<AnnotationFS> before = TextRulerToolkit.getAnnotationsBeforePosition( |
| example.getDocumentCAS(), slotAnnotation.getBegin(), -1, |
| TextRulerToolkit.getFilterSetWithSlotNames(slotNames, filterSet), tokensRootType); |
| List<AnnotationFS> after = TextRulerToolkit.getAnnotationsAfterPosition( |
| example.getDocumentCAS(), slotAnnotation.getEnd(), -1, |
| TextRulerToolkit.getFilterSetWithSlotNames(slotNames, filterSet), tokensRootType); |
| List<AnnotationFS> inside = TextRulerToolkit.getAnnotationsWithinBounds( |
| example.getDocumentCAS(), slotAnnotation.getBegin(), slotAnnotation.getEnd(), |
| TextRulerToolkit.getFilterSetWithSlotNames(slotNames, filterSet), tokensRootType); |
| |
| // the before annotations have to be reversed: |
| for (int i = before.size() - 1; i >= 0; i--) { |
| AnnotationFS afs = before.get(i); |
| RapierRuleItem ruleItem = new RapierRuleItem(); |
| ruleItem.addWordConstraint(new TextRulerWordConstraint(new TextRulerAnnotation(afs, example |
| .getDocument()))); |
| addAvailablePosTagConstraintToItem(ruleItem, afs, example); |
| rule.addPreFillerItem(ruleItem); |
| } |
| |
| for (AnnotationFS afs : inside) { |
| RapierRuleItem ruleItem = new RapierRuleItem(); |
| ruleItem.addWordConstraint(new TextRulerWordConstraint(new TextRulerAnnotation(afs, example |
| .getDocument()))); |
| addAvailablePosTagConstraintToItem(ruleItem, afs, example); |
| rule.addFillerItem(ruleItem); |
| } |
| for (AnnotationFS afs : after) { |
| RapierRuleItem ruleItem = new RapierRuleItem(); |
| ruleItem.addWordConstraint(new TextRulerWordConstraint(new TextRulerAnnotation(afs, example |
| .getDocument()))); |
| addAvailablePosTagConstraintToItem(ruleItem, afs, example); |
| rule.addPostFillerItem(ruleItem); |
| } |
| |
| // TextRulerToolkit.log("RULE: "+rule.getRuleString()); |
| // testRuleOnTrainingsSet(rule, exampleDocuments.getDocuments()); |
| |
| // this rule has to at least cover its seed example!! |
| TextRulerStatisticsCollector c = new TextRulerStatisticsCollector(); |
| c.addCoveredPositive(example); |
| rule.setCoveringStatistics(c); |
| slotRules.add(rule); |
| } |
| initialRuleBaseSize = slotRules.size(); |
| } |
| |
| protected void addRuleAndRemoveEmpiricallySubsumedRules(RapierRule rule) { |
| if (!slotRules.contains(rule)) { |
| List<TextRulerRule> rulesToRemove = new ArrayList<TextRulerRule>(); |
| Set<TextRulerExample> coveredExamples = rule.getCoveringStatistics() |
| .getCoveredPositiveExamples(); |
| for (TextRulerRule r : slotRules) { |
| if (coveredExamples.containsAll(r.getCoveringStatistics().getCoveredPositiveExamples())) |
| rulesToRemove.add(r); |
| } |
| for (TextRulerRule removeR : rulesToRemove) |
| slotRules.remove(removeR); |
| slotRules.add(rule); |
| updateCompressionStatusString(); |
| } |
| } |
| |
| protected RapierRule findNewRule() { |
| Random rand = new Random(System.currentTimeMillis()); |
| |
| Set<RapierRule> generalizations = new HashSet<RapierRule>(); |
| // 0. initialization |
| ruleList.clear(); |
| |
| if (slotRules.size() <= 1) |
| return null; |
| |
| List<RapierRule> uncompressedRules = new ArrayList<RapierRule>(); |
| for (TextRulerRule r : slotRules) { |
| if (((RapierRule) r).isInitialRule()) |
| uncompressedRules.add((RapierRule) r); |
| } |
| |
| // 1. get generalizations of the two slot filler patterns: |
| |
| // create pairs and prefer still uncompressed rules when choosing |
| // "randomly": |
| int pairsLeft = pairCount; |
| if (uncompressedRules.size() == 1) { |
| RapierRule rule1 = uncompressedRules.get(0); |
| RapierRule rule2 = null; |
| while (rule2 == null || rule1 == rule2) { |
| rule2 = (RapierRule) slotRules.get(rand.nextInt(slotRules.size())); |
| } |
| generalizations.addAll(getFillerGeneralizationsForRulePair(rule1, rule2)); |
| if (shouldAbort()) |
| return null; |
| pairsLeft--; |
| } else if (uncompressedRules.size() == 2) { |
| RapierRule rule1 = uncompressedRules.get(0); |
| RapierRule rule2 = uncompressedRules.get(1); |
| generalizations.addAll(getFillerGeneralizationsForRulePair(rule1, rule2)); |
| if (shouldAbort()) |
| return null; |
| pairsLeft--; |
| } else if (uncompressedRules.size() > 2) { |
| int uPairCount = pairCount; |
| if (uPairCount > uncompressedRules.size()) |
| uPairCount /= 2; |
| for (int i = 0; i < uPairCount; i++) { |
| RapierRule rule1 = uncompressedRules.get(rand.nextInt(uncompressedRules.size())); |
| RapierRule rule2 = null; |
| while (rule2 == null || rule1 == rule2) { |
| rule2 = uncompressedRules.get(rand.nextInt(uncompressedRules.size())); |
| } |
| generalizations.addAll(getFillerGeneralizationsForRulePair(rule1, rule2)); |
| pairsLeft--; |
| } |
| } |
| |
| for (int i = 0; i < pairsLeft; i++) { |
| // TODO optimize !! don't call the machinery with the same rule pair |
| // two times in one session !!! |
| // randomly pick two rules: |
| RapierRule rule1 = (RapierRule) slotRules.get(rand.nextInt(slotRules.size())); |
| RapierRule rule2 = null; |
| while (rule2 == null || rule1 == rule2) { |
| rule2 = (RapierRule) slotRules.get(rand.nextInt(slotRules.size())); |
| } |
| generalizations.addAll(getFillerGeneralizationsForRulePair(rule1, rule2)); |
| |
| if (shouldAbort()) |
| return null; |
| } |
| |
| // if (TextRulerToolkit.DEBUG) |
| // { |
| // TextRulerToolkit.log("Rule Generalizations created: " + |
| // generalizations.size()); |
| // for (RapierRule newRule : generalizations) |
| // TextRulerToolkit.log("Rule = "+newRule.getRuleString()); |
| // } |
| |
| // 2. evaluate an enque to priority list: |
| List<RapierRule> testRules = new ArrayList<RapierRule>(generalizations); |
| |
| for (RapierRule r : testRules) { |
| r.combineSenselessPatternListItems(); |
| } |
| |
| testRulesIfNotCached(testRules); |
| if (shouldAbort()) |
| return null; |
| |
| for (RapierRule newRule : generalizations) { |
| if (TextRulerToolkit.DEBUG) { |
| if (!RapierDebugHelper.debugCheckIfRuleCoversItsSeedRuleCoverings(newRule)) { |
| TextRulerToolkit |
| .log("------------------------------------------------------------------------------------------"); |
| TextRulerToolkit |
| .log("ERROR, A RULE HAS TO COVER AT LEAST EVERY POSITIVE EXAMPLE OF ITS TWO SEED RULES!!!"); |
| TextRulerToolkit.log("\t RULE: " + newRule.getRuleString()); |
| TextRulerToolkit.log("\t Parent1: " + newRule.getParent1().getRuleString()); |
| TextRulerToolkit.log("\t Parent2: " + newRule.getParent2().getRuleString()); |
| TextRulerToolkit.log("--------"); |
| TextRulerToolkit.log("+RuleCovering: " |
| + newRule.getCoveringStatistics().getCoveredPositiveExamples()); |
| TextRulerToolkit.log("+P1Covering : " |
| + newRule.getParent1().getCoveringStatistics().getCoveredPositiveExamples()); |
| TextRulerToolkit.log("+P2Covering : " |
| + newRule.getParent2().getCoveringStatistics().getCoveredPositiveExamples()); |
| |
| } |
| } |
| ruleList.add(newRule); |
| } |
| |
| // 3. specialize pre and post fillers: |
| int n = 0; |
| double bestValue = Double.MAX_VALUE; |
| int noImprovementCounter = 0; |
| while (true) { |
| n++; |
| TextRulerToolkit.log(" --- NEW SPECIALIZATOIN ROUND; n = " + n + " noImprovementCounter = " |
| + noImprovementCounter); |
| List<RapierRule> newRuleList = new ArrayList<RapierRule>(); |
| for (RapierRule curRule : ruleList) { |
| |
| List<RapierRule> specTestRules = new ArrayList<RapierRule>(specializePreFiller(curRule, n)); |
| |
| for (RapierRule r : specTestRules) |
| r.combineSenselessPatternListItems(); |
| |
| testRulesIfNotCached(specTestRules); |
| if (shouldAbort()) |
| return null; |
| |
| for (RapierRule r : specTestRules) |
| newRuleList.add(r); |
| } |
| ruleList.addAll(newRuleList); |
| |
| newRuleList.clear(); |
| for (RapierRule curRule : ruleList) { |
| |
| List<RapierRule> specTestRules = new ArrayList<RapierRule>(specializePostFiller(curRule, n)); |
| |
| for (RapierRule r : specTestRules) |
| r.combineSenselessPatternListItems(); |
| |
| testRulesIfNotCached(specTestRules); |
| if (shouldAbort()) |
| return null; |
| |
| for (RapierRule r : specTestRules) |
| newRuleList.add(r); |
| } |
| ruleList.addAll(newRuleList); |
| |
| RapierRule bestRule = ruleList.peek(); |
| |
| if (TextRulerToolkit.DEBUG) { |
| // for (RapierRule r: ruleList) |
| // TextRulerToolkit.log("value="+r.getPriority()+" rule = "+r.getRuleString()); |
| TextRulerToolkit.log("------------------------------------"); |
| TextRulerToolkit.log("BEST RULE FOR THIS SESSION: " + bestRule.getCoveringStatistics()); |
| TextRulerToolkit.log(bestRule.getRuleString()); |
| TextRulerToolkit.log("------------------------------------"); |
| } |
| if (bestRule.producesOnlyValidFillers()) |
| break; // todo: horizon effects ?? |
| |
| if (bestRule.getPriority() < bestValue) { |
| noImprovementCounter = 0; |
| bestValue = bestRule.getPriority(); |
| } else { |
| noImprovementCounter++; |
| if (noImprovementCounter > limNoImprovements) |
| break; |
| } |
| } |
| |
| RapierRule bestRule = ruleList.peek(); |
| return bestRule; |
| } |
| |
| private List<RapierRule> getFillerGeneralizationsForRulePair(RapierRule rule1, RapierRule rule2) { |
| TextRulerToolkit |
| .log("------------------------------------------------------------------------------------------"); |
| TextRulerToolkit.log("getFillerGeneralizationsForRulePair:"); |
| TextRulerToolkit.log("Rule1: " + rule1.getRuleString()); |
| TextRulerToolkit.log("Rule2: " + rule2.getRuleString()); |
| |
| List<RapierRule> result = new ArrayList<RapierRule>(); |
| List<TextRulerRulePattern> genList = RapierGeneralizationHelper |
| .getGeneralizationsForRuleItemPatterns(rule1.getFillerPattern(), |
| rule2.getFillerPattern()); |
| // create rules: |
| for (TextRulerRulePattern pattern : genList) { |
| RapierRule newRule = new RapierRule(this, rule1.getTarget()); |
| for (TextRulerRuleItem patternItem : pattern) |
| newRule.addFillerItem(patternItem.copy()); |
| newRule.setParent1(rule1.copy()); |
| newRule.setParent1PreFiller_n(0); |
| newRule.setParent1PostFiller_n(0); |
| newRule.setParent2(rule2.copy()); |
| newRule.setParent2PreFiller_n(0); |
| newRule.setParent2PostFiller_n(0); |
| result.add(newRule); |
| newRule.setNeedsCompile(true); |
| // TextRulerToolkit.log("newRule: "+newRule.getRuleString()); |
| } |
| TextRulerToolkit.log(" getGeneralizationsForRulePair result list size = " + result.size()); |
| return result; |
| } |
| |
| public List<RapierRule> specializePreFiller(RapierRule curRule, int n) { |
| RapierRule baseRule1 = curRule.getParent1(); |
| RapierRule baseRule2 = curRule.getParent2(); |
| int n1 = curRule.getParent1PreFiller_n(); |
| int n2 = curRule.getParent2PreFiller_n(); |
| TextRulerRulePattern preFiller1 = baseRule1.getPreFillerPattern(); |
| TextRulerRulePattern preFiller2 = baseRule2.getPreFillerPattern(); |
| int preFiller1MaxIndex = preFiller1.size() - n1 - 1; |
| int preFiller2MaxIndex = preFiller2.size() - n2 - 1; |
| |
| // generate 3 different possible sets for generalizations: |
| |
| // 1. n vs. n-1 (n elements of baserule1, n-1 of baserule2) |
| TextRulerRulePattern consideredPreFiller1 = new TextRulerRulePattern(); |
| TextRulerRulePattern consideredPreFiller2 = new TextRulerRulePattern(); |
| for (int i = preFiller1.size() - n; i >= 0 && i <= preFiller1MaxIndex; i++) |
| consideredPreFiller1.add(preFiller1.get(i)); |
| for (int i = preFiller2.size() - n + 1; i >= 0 && i <= preFiller2MaxIndex; i++) |
| consideredPreFiller2.add(preFiller2.get(i)); |
| List<TextRulerRulePattern> genList1 = null; |
| if (consideredPreFiller1.size() + consideredPreFiller2.size() > 0) |
| genList1 = RapierGeneralizationHelper.getGeneralizationsForRuleItemPatterns( |
| consideredPreFiller1, consideredPreFiller2); |
| |
| List<TextRulerRulePattern> genList2 = null; |
| List<TextRulerRulePattern> genList3 = null; |
| |
| if (useAllGenSetsAtSpecialization) // due to performance reasons the |
| // user can switch this off |
| { |
| // 2. n-1 vs. n (n-1 elements of baserule1, n of baserule2) |
| consideredPreFiller1.clear(); |
| consideredPreFiller2.clear(); |
| for (int i = preFiller1.size() - n + 1; i >= 0 && i <= preFiller1MaxIndex; i++) |
| consideredPreFiller1.add(preFiller1.get(i)); |
| for (int i = preFiller2.size() - n; i >= 0 && i <= preFiller2MaxIndex; i++) |
| consideredPreFiller2.add(preFiller2.get(i)); |
| |
| if (consideredPreFiller1.size() + consideredPreFiller2.size() > 0) |
| genList2 = RapierGeneralizationHelper.getGeneralizationsForRuleItemPatterns( |
| consideredPreFiller1, consideredPreFiller2); |
| |
| // 3. n vs. n (n elements of baserule1, n of baserule2) |
| consideredPreFiller1.clear(); |
| consideredPreFiller2.clear(); |
| for (int i = preFiller1.size() - n; i >= 0 && i <= preFiller1MaxIndex; i++) |
| consideredPreFiller1.add(preFiller1.get(i)); |
| for (int i = preFiller2.size() - n; i >= 0 && i <= preFiller2MaxIndex; i++) |
| consideredPreFiller2.add(preFiller2.get(i)); |
| if (consideredPreFiller1.size() + consideredPreFiller2.size() > 0) |
| genList3 = RapierGeneralizationHelper.getGeneralizationsForRuleItemPatterns( |
| consideredPreFiller1, consideredPreFiller2); |
| } |
| |
| // TODO optimize and don't store all 3 genLists ! but for debugging |
| // purposes we keep them for now ! |
| Set<TextRulerRulePattern> genSet = new HashSet<TextRulerRulePattern>(); |
| if (genList1 != null) |
| genSet.addAll(genList1); |
| if (genList2 != null) |
| genSet.addAll(genList2); |
| if (genList3 != null) |
| genSet.addAll(genList3); |
| |
| List<RapierRule> resultRules = new ArrayList<RapierRule>(); |
| |
| for (TextRulerRulePattern l : genSet) { |
| RapierRule newRule = curRule.copy(); |
| for (int i = l.size() - 1; i >= 0; i--) |
| newRule.addPreFillerItem(l.get(i)); |
| newRule.setParent1PreFiller_n(n); |
| newRule.setParent2PreFiller_n(n); |
| resultRules.add(newRule); |
| } |
| return resultRules; |
| } |
| |
| // n = 1..maxN |
| public List<RapierRule> specializePostFiller(RapierRule curRule, int n) { |
| if (n == 0) { |
| TextRulerToolkit.log("ERROR ! N SHOULD NOT BE 0!"); |
| } |
| RapierRule baseRule1 = curRule.getParent1(); |
| RapierRule baseRule2 = curRule.getParent2(); |
| int n1 = curRule.getParent1PostFiller_n(); |
| int n2 = curRule.getParent2PostFiller_n(); |
| TextRulerRulePattern postFiller1 = baseRule1.getPostFillerPattern(); |
| TextRulerRulePattern postFiller2 = baseRule2.getPostFillerPattern(); |
| int postFiller1MinIndex = n1; |
| int postFiller2MinIndex = n2; |
| |
| // generate 3 different possible sets for generalizations: |
| |
| // 1. n vs. n-1 (n elements of baserule1, n-1 of baserule2) |
| TextRulerRulePattern consideredPostFiller1 = new TextRulerRulePattern(); |
| TextRulerRulePattern consideredPostFiller2 = new TextRulerRulePattern(); |
| for (int i = postFiller1MinIndex; i < postFiller1.size() && i < n; i++) |
| consideredPostFiller1.add(postFiller1.get(i)); |
| for (int i = postFiller2MinIndex; i < postFiller2.size() && i < n - 1; i++) |
| consideredPostFiller2.add(postFiller2.get(i)); |
| List<TextRulerRulePattern> genList1 = null; |
| if (consideredPostFiller1.size() + consideredPostFiller2.size() > 0) |
| genList1 = RapierGeneralizationHelper.getGeneralizationsForRuleItemPatterns( |
| consideredPostFiller1, consideredPostFiller2); |
| |
| // 2. n-1 vs. n (n-1 elements of baserule1, n of baserule2) |
| consideredPostFiller1.clear(); |
| consideredPostFiller2.clear(); |
| for (int i = postFiller1MinIndex; i < postFiller1.size() && i < n - 1; i++) |
| consideredPostFiller1.add(postFiller1.get(i)); |
| for (int i = postFiller2MinIndex; i < postFiller2.size() && i < n; i++) |
| consideredPostFiller2.add(postFiller2.get(i)); |
| List<TextRulerRulePattern> genList2 = null; |
| if (consideredPostFiller1.size() + consideredPostFiller2.size() > 0) |
| genList2 = RapierGeneralizationHelper.getGeneralizationsForRuleItemPatterns( |
| consideredPostFiller1, consideredPostFiller2); |
| |
| // 3. n vs. n (n elements of baserule1, n of baserule2) |
| consideredPostFiller1.clear(); |
| consideredPostFiller2.clear(); |
| for (int i = postFiller1MinIndex; i < postFiller1.size() && i < n; i++) |
| consideredPostFiller1.add(postFiller1.get(i)); |
| for (int i = postFiller2MinIndex; i < postFiller2.size() && i < n; i++) |
| consideredPostFiller2.add(postFiller2.get(i)); |
| List<TextRulerRulePattern> genList3 = null; |
| if (consideredPostFiller1.size() + consideredPostFiller2.size() > 0) |
| genList3 = RapierGeneralizationHelper.getGeneralizationsForRuleItemPatterns( |
| consideredPostFiller1, consideredPostFiller2); |
| |
| // TODO optimize and don't store all 3 genLists ! but for debugging |
| // purposes we keep them for now ! |
| Set<TextRulerRulePattern> genSet = new HashSet<TextRulerRulePattern>(); |
| if (genList1 != null) |
| genSet.addAll(genList1); |
| if (genList2 != null) |
| genSet.addAll(genList2); |
| if (genList3 != null) |
| genSet.addAll(genList3); |
| |
| List<RapierRule> resultRules = new ArrayList<RapierRule>(); |
| |
| for (TextRulerRulePattern l : genSet) { |
| RapierRule newRule = curRule.copy(); |
| for (TextRulerRuleItem t : l) |
| newRule.addPostFillerItem(t); |
| newRule.setParent1PostFiller_n(n); |
| newRule.setParent2PostFiller_n(n); |
| resultRules.add(newRule); |
| } |
| return resultRules; |
| } |
| |
| @Override |
| public boolean collectNegativeCoveredInstancesWhenTesting() { |
| return false; |
| } |
| |
| public String getResultString() { |
| if (slotRules != null) |
| return slotRules.getTMFileString(getFileHeaderString(true), 1000); // if |
| // a |
| // rule |
| // is |
| // >100 |
| // characters, |
| // it |
| // gets |
| // replaced |
| // by |
| // a |
| // placeholder |
| else |
| return "No results available yet!"; |
| } |
| |
| public void setParameters(Map<String, Object> params) { |
| if (TextRulerToolkit.DEBUG) |
| saveParametersToTempFolder(params); |
| |
| // TODO add try catch |
| if (params.containsKey(COMPRESSION_FAIL_MAX_COUNT_KEY)) |
| compressionFailMaxCount = (Integer) params.get(COMPRESSION_FAIL_MAX_COUNT_KEY); |
| |
| if (params.containsKey(RULELIST_SIZE_KEY)) |
| ruleListSize = (Integer) params.get(RULELIST_SIZE_KEY); |
| |
| if (params.containsKey(PAIR_COUNT_KEY)) |
| pairCount = (Integer) params.get(PAIR_COUNT_KEY); |
| |
| if (params.containsKey(LIM_NO_IMPROVEMENTS_KEY)) |
| limNoImprovements = (Integer) params.get(LIM_NO_IMPROVEMENTS_KEY); |
| |
| if (params.containsKey(NOISE_THESHOLD_KEY)) |
| noiseThreshold = (Float) params.get(NOISE_THESHOLD_KEY); |
| |
| if (params.containsKey(POSTAG_ROOTTYPE_KEY)) |
| posTagRootTypeName = (String) params.get(POSTAG_ROOTTYPE_KEY); |
| |
| if (params.containsKey(MIN_COVERED_POSITIVES_KEY)) |
| minCoveredPositives = (Integer) params.get(MIN_COVERED_POSITIVES_KEY); |
| |
| if (params.containsKey(USE_ALL_GENSETS_AT_SPECIALIZATION_KEY)) |
| useAllGenSetsAtSpecialization = (Boolean) params.get(USE_ALL_GENSETS_AT_SPECIALIZATION_KEY); |
| } |
| |
| // TODO share this between algorithms (e.g. LP2 and RAPIER ?) and make a |
| // maximum size of the cache, etc. like CasCache? |
| protected void testRulesIfNotCached(List<RapierRule> rules) { |
| List<TextRulerRule> rulesToTest = new ArrayList<TextRulerRule>(); |
| |
| for (RapierRule r : rules) { |
| String key = r.getRuleString(); |
| if (cachedTestedRuleStatistics.containsKey(key)) { |
| r.setCoveringStatistics(cachedTestedRuleStatistics.get(key).copy()); |
| TextRulerToolkit.log("CACHE HIT; size=" + cachedTestedRuleStatistics.size()); |
| } else |
| rulesToTest.add(r); |
| } |
| |
| if (rulesToTest.size() > 0) { |
| testRulesOnDocumentSet(rulesToTest, exampleDocuments); |
| if (shouldAbort()) |
| return; |
| while (cachedTestedRuleStatistics.size() + rulesToTest.size() > 10000) // TODO |
| // lohnt |
| // sich |
| // das |
| // ? |
| // speicher |
| // beobachten |
| // !! |
| { |
| Iterator<String> it = cachedTestedRuleStatistics.keySet().iterator(); |
| if (!it.hasNext()) |
| break; |
| String removeKey = cachedTestedRuleStatistics.keySet().iterator().next(); |
| cachedTestedRuleStatistics.remove(removeKey); |
| } |
| |
| for (TextRulerRule r : rulesToTest) { |
| String key = r.getRuleString(); |
| cachedTestedRuleStatistics.put(key, r.getCoveringStatistics().copy()); |
| } |
| } |
| } |
| |
| } |
