| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.stanbol.enhancer.engines.opennlp.impl; |
| |
| import static org.apache.stanbol.enhancer.nlp.NlpAnnotations.NER_ANNOTATION; |
| import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_RELATION; |
| import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.DC_TYPE; |
| import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_CONFIDENCE; |
| import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_END; |
| import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTED_TEXT; |
| import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_SELECTION_CONTEXT; |
| import static org.apache.stanbol.enhancer.servicesapi.rdf.Properties.ENHANCER_START; |
| |
| import java.io.IOException; |
| import java.nio.charset.Charset; |
| import java.util.ArrayList; |
| import java.util.Collection; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.Iterator; |
| import java.util.LinkedHashMap; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| |
| import opennlp.tools.namefind.NameFinderME; |
| import opennlp.tools.namefind.TokenNameFinderModel; |
| import opennlp.tools.sentdetect.SentenceDetectorME; |
| import opennlp.tools.sentdetect.SentenceModel; |
| import opennlp.tools.tokenize.Tokenizer; |
| import opennlp.tools.util.InvalidFormatException; |
| import opennlp.tools.util.Span; |
| |
| import org.apache.clerezza.commons.rdf.Language; |
| import org.apache.clerezza.rdf.core.LiteralFactory; |
| import org.apache.clerezza.commons.rdf.Graph; |
| import org.apache.clerezza.commons.rdf.IRI; |
| import org.apache.clerezza.commons.rdf.impl.utils.PlainLiteralImpl; |
| import org.apache.clerezza.commons.rdf.impl.utils.TripleImpl; |
| import org.apache.commons.lang.StringUtils; |
| import org.apache.stanbol.commons.opennlp.OpenNLP; |
| import org.apache.stanbol.commons.stanboltools.datafileprovider.DataFileProvider; |
| import org.apache.stanbol.enhancer.nlp.NlpAnnotations; |
| import org.apache.stanbol.enhancer.nlp.model.AnalysedText; |
| import org.apache.stanbol.enhancer.nlp.model.AnalysedTextUtils; |
| import org.apache.stanbol.enhancer.nlp.model.Chunk; |
| import org.apache.stanbol.enhancer.nlp.model.Section; |
| import org.apache.stanbol.enhancer.nlp.model.Sentence; |
| import org.apache.stanbol.enhancer.nlp.model.Token; |
| import org.apache.stanbol.enhancer.nlp.model.annotation.Value; |
| import org.apache.stanbol.enhancer.nlp.ner.NerTag; |
| import org.apache.stanbol.enhancer.servicesapi.Blob; |
| import org.apache.stanbol.enhancer.servicesapi.ContentItem; |
| import org.apache.stanbol.enhancer.servicesapi.EngineException; |
| import org.apache.stanbol.enhancer.servicesapi.EnhancementEngine; |
| import org.apache.stanbol.enhancer.servicesapi.InvalidContentException; |
| import org.apache.stanbol.enhancer.servicesapi.helper.ContentItemHelper; |
| import org.apache.stanbol.enhancer.servicesapi.helper.EnhancementEngineHelper; |
| import org.apache.stanbol.enhancer.servicesapi.impl.AbstractEnhancementEngine; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| /** |
| * Core of the NER EnhancementEngine(s), separated from the OSGi service to make |
| * it easier to test this. |
| */ |
| public abstract class NEREngineCore |
| extends AbstractEnhancementEngine<IOException,RuntimeException> |
| implements EnhancementEngine { |
| protected static final String TEXT_PLAIN_MIMETYPE = "text/plain"; |
| /** |
| * Contains the only supported mimetype {@link #TEXT_PLAIN_MIMETYPE} |
| */ |
| protected static final Set<String> SUPPORTED_MIMETYPES = |
| Collections.singleton(TEXT_PLAIN_MIMETYPE); |
| |
| private final Logger log = LoggerFactory.getLogger(getClass()); |
| |
| protected OpenNLP openNLP; |
| |
| protected NEREngineConfig config; |
| |
| |
| /** Comments about our models */ |
| public static final Map<String, String> DATA_FILE_COMMENTS; |
| static { |
| DATA_FILE_COMMENTS = new HashMap<String, String>(); |
| DATA_FILE_COMMENTS.put("Default data files", "provided by the org.apache.stanbol.defaultdata bundle"); |
| } |
| /** |
| * If used sub classes MUST ensure that {@link #openNLP} and {@link #config} |
| * are set before calling {@link #canEnhance(ContentItem)} or |
| * {@link #computeEnhancements(ContentItem)} |
| */ |
| protected NEREngineCore(){} |
| |
| NEREngineCore(OpenNLP openNLP, NEREngineConfig config) throws InvalidFormatException, IOException{ |
| if(openNLP == null){ |
| throw new IllegalArgumentException("The parsed OpenNLP instance MUST NOT be NULL!"); |
| } |
| if(config == null){ |
| throw new IllegalArgumentException("The parsed NER engine configuration MUST NOT be NULL!"); |
| } |
| this.openNLP = openNLP; |
| this.config = config; |
| } |
| |
| NEREngineCore(DataFileProvider dfp,NEREngineConfig config) throws InvalidFormatException, IOException { |
| this(new OpenNLP(dfp),config); |
| } |
| |
| |
| public void computeEnhancements(ContentItem ci) throws EngineException { |
| //first check the langauge before processing the content (text) |
| String language = extractLanguage(ci); |
| if(language == null){ |
| throw new IllegalStateException("Unable to extract Language for " |
| + "ContentItem "+ci.getUri()+": This is also checked in the canEnhance " |
| + "method! -> This indicated an Bug in the implementation of the " |
| + "EnhancementJobManager!"); |
| } |
| if(!isNerModel(language)){ |
| throw new IllegalStateException("For the language '"+language+"' of ContentItem "+ci.getUri() |
| + " no NER model is configured: This is also checked in the canEnhance " |
| + "method! -> This indicated an Bug in the implementation of the " |
| + "EnhancementJobManager!"); |
| } |
| final AnalysedText at = AnalysedTextUtils.getAnalysedText(ci); |
| //validate data in the AnalysedText |
| final String text; |
| if(at != null && at.getTokens().hasNext()){ //if the AnalysedText is present and tokens are present |
| if(log.isDebugEnabled()){ |
| log.debug("computeEnhancements from AnalysedText ContentPart of ContentItem {}: text={}", |
| ci.getUri().getUnicodeString(), StringUtils.abbreviate(at.getSpan(), 100)); |
| } |
| text = null; |
| } else { //no AnalysedText with tokens ... |
| //fallback to processing the plain text is still supported |
| Entry<IRI,Blob> contentPart = ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES); |
| if(contentPart == null){ |
| throw new IllegalStateException("No ContentPart with Mimetype '" |
| + TEXT_PLAIN_MIMETYPE+"' found for ContentItem "+ci.getUri() |
| + ": This is also checked in the canEnhance method! -> This " |
| + "indicated an Bug in the implementation of the " |
| + "EnhancementJobManager!"); |
| } |
| try { |
| text = ContentItemHelper.getText(contentPart.getValue()); |
| } catch (IOException e) { |
| throw new InvalidContentException(this, ci, e); |
| } |
| if (text.trim().length() == 0) { |
| // TODO: make the length of the data a field of the ContentItem |
| // interface to be able to filter out empty items in the canEnhance |
| // method |
| log.warn("ContentPart {} of ContentItem {} does not contain any text" + |
| "to extract knowledge from in ContentItem {}", |
| contentPart.getKey(),ci); |
| return; |
| } |
| if(log.isDebugEnabled()){ |
| log.debug("computeEnhancements from ContentPart {} of ContentItem {}: text={}", |
| new Object[]{contentPart.getKey(),ci.getUri().getUnicodeString(), |
| StringUtils.abbreviate(text, 100)}); |
| } |
| } |
| try { |
| if(config.isProcessedLangage(language)){ |
| for (String defaultModelType : config.getDefaultModelTypes()) { |
| TokenNameFinderModel nameFinderModel = openNLP.getNameModel(defaultModelType, language); |
| if(nameFinderModel == null){ |
| log.info("No NER Model for {} and language {} available!",defaultModelType,language); |
| } else { |
| findNamedEntities(ci, at, text, language, nameFinderModel); |
| } |
| } |
| } //else do not use default models for languages other than the processed one |
| //process for additional models |
| for(String additionalModel : config.getSpecificNerModles(language)){ |
| TokenNameFinderModel nameFinderModel; |
| try { |
| nameFinderModel = openNLP.getModel(TokenNameFinderModel.class, |
| additionalModel, null); |
| findNamedEntities(ci, at, text, language, nameFinderModel); |
| } catch (IOException e) { |
| log.warn("Unable to load TokenNameFinderModel model for language '"+language |
| + "' (model: "+additionalModel+")",e); |
| } catch (RuntimeException e){ |
| log.warn("Error while creating ChunkerModel for language '"+language |
| + "' (model: "+additionalModel+")",e); |
| } |
| } |
| } catch (Exception e) { |
| if (e instanceof RuntimeException) { |
| throw (RuntimeException)e; |
| } else { |
| throw new EngineException(this, ci, e); |
| } |
| } |
| } |
| |
| protected void findNamedEntities(final ContentItem ci, |
| final AnalysedText at, |
| final String text, |
| final String lang, |
| final TokenNameFinderModel nameFinderModel) { |
| |
| if (ci == null) { |
| throw new IllegalArgumentException("Parsed ContentItem MUST NOT be NULL"); |
| } |
| if (at == null && text == null) { |
| log.warn("NULL was parsed as AnalysedText AND Text for content item " |
| + ci.getUri() + ". One of the two MUST BE present! -> call ignored"); |
| return; |
| } |
| final Language language; |
| if(lang != null && !lang.isEmpty()){ |
| language = new Language(lang); |
| } else { |
| language = null; |
| } |
| if(log.isDebugEnabled()){ |
| log.debug("findNamedEntities model={}, language={}, text=", |
| new Object[]{ nameFinderModel, language, |
| StringUtils.abbreviate(at != null ? at.getSpan() : text, 100) }); |
| } |
| LiteralFactory literalFactory = LiteralFactory.getInstance(); |
| Graph g = ci.getMetadata(); |
| Map<String,List<NameOccurrence>> entityNames; |
| if(at != null){ |
| entityNames = extractNameOccurrences(nameFinderModel, at, lang); |
| } else { |
| entityNames = extractNameOccurrences(nameFinderModel, text,lang); |
| } |
| //lock the ContentItem while writing the RDF data for found Named Entities |
| ci.getLock().writeLock().lock(); |
| try { |
| Map<String,IRI> previousAnnotations = new LinkedHashMap<String,IRI>(); |
| for (Map.Entry<String,List<NameOccurrence>> nameInContext : entityNames.entrySet()) { |
| |
| String name = nameInContext.getKey(); |
| List<NameOccurrence> occurrences = nameInContext.getValue(); |
| |
| IRI firstOccurrenceAnnotation = null; |
| |
| for (NameOccurrence occurrence : occurrences) { |
| IRI textAnnotation = EnhancementEngineHelper.createTextEnhancement(ci, this); |
| g.add(new TripleImpl(textAnnotation, ENHANCER_SELECTED_TEXT, |
| new PlainLiteralImpl(name, language))); |
| g.add(new TripleImpl(textAnnotation, ENHANCER_SELECTION_CONTEXT, |
| new PlainLiteralImpl(occurrence.context, language))); |
| if(occurrence.type != null){ |
| g.add(new TripleImpl(textAnnotation, DC_TYPE, occurrence.type)); |
| } |
| if(occurrence.confidence != null){ |
| g.add(new TripleImpl(textAnnotation, ENHANCER_CONFIDENCE, literalFactory |
| .createTypedLiteral(occurrence.confidence))); |
| } |
| if (occurrence.start != null && occurrence.end != null) { |
| g.add(new TripleImpl(textAnnotation, ENHANCER_START, literalFactory |
| .createTypedLiteral(occurrence.start))); |
| g.add(new TripleImpl(textAnnotation, ENHANCER_END, literalFactory |
| .createTypedLiteral(occurrence.end))); |
| } |
| |
| // add the subsumption relationship among occurrences of the same |
| // name |
| if (firstOccurrenceAnnotation == null) { |
| // check already extracted annotations to find a first most |
| // specific occurrence |
| for (Map.Entry<String,IRI> entry : previousAnnotations.entrySet()) { |
| if (entry.getKey().contains(name)) { |
| // we have found a most specific previous |
| // occurrence, use it as subsumption target |
| firstOccurrenceAnnotation = entry.getValue(); |
| g.add(new TripleImpl(textAnnotation, DC_RELATION, firstOccurrenceAnnotation)); |
| break; |
| } |
| } |
| if (firstOccurrenceAnnotation == null) { |
| // no most specific previous occurrence, I am the first, |
| // most specific occurrence to be later used as a target |
| firstOccurrenceAnnotation = textAnnotation; |
| previousAnnotations.put(name, textAnnotation); |
| } |
| } else { |
| // I am referring to a most specific first occurrence of the |
| // same name |
| g.add(new TripleImpl(textAnnotation, DC_RELATION, firstOccurrenceAnnotation)); |
| } |
| } |
| } |
| } finally { |
| ci.getLock().writeLock().unlock(); |
| } |
| } |
| |
| @Deprecated |
| public Collection<String> extractPersonNames(String text) { |
| return extractPersonNames(text, "en"); |
| } |
| public Collection<String> extractPersonNames(String text,String lang) { |
| return extractNames(getNameModel("person",lang),text); |
| } |
| |
| @Deprecated |
| public Collection<String> extractLocationNames(String text) { |
| return extractLocationNames(text,"en"); |
| } |
| |
| public Collection<String> extractLocationNames(String text,String lang) { |
| return extractNames(getNameModel("location",lang), text); |
| } |
| |
| @Deprecated |
| public Collection<String> extractOrganizationNames(String text) { |
| return extractOrganizationNames(text,"en"); |
| } |
| public Collection<String> extractOrganizationNames(String text,String lang) { |
| return extractNames(getNameModel("organization",lang), text); |
| } |
| /** |
| * extracts the PersonName occurrences for English language texts |
| * @param text |
| * @return |
| * @deprecated use {@link #extractLocationNameOccurrences(String,String)} instead |
| */ |
| @Deprecated |
| public Map<String,List<NameOccurrence>> extractPersonNameOccurrences(String text) { |
| return this.extractPersonNameOccurrences(text, "en"); |
| } |
| public Map<String,List<NameOccurrence>> extractPersonNameOccurrences(String text, String lang) { |
| return extractNameOccurrences(getNameModel("person",lang), text, lang); |
| } |
| /** |
| * extracts the LocationName occurrences for English language texts |
| * @param text |
| * @return |
| * @deprecated use {@link #extractLocationNameOccurrences(String,String)} instead |
| */ |
| @Deprecated |
| public Map<String,List<NameOccurrence>> extractLocationNameOccurrences(String text) { |
| return extractLocationNameOccurrences(text, "en"); |
| } |
| |
| public Map<String,List<NameOccurrence>> extractLocationNameOccurrences(String text,String lang) { |
| return extractNameOccurrences(getNameModel("location",lang), text,lang); |
| } |
| |
| /** |
| * extracts the OrganizationName occurrences for English language texts |
| * @param text |
| * @return |
| * @deprecated use {@link #extractOrganizationNamesOccurrences(String,String)} instead |
| */ |
| @Deprecated |
| public Map<String,List<NameOccurrence>> extractOrganizationNameOccurrences(String text) { |
| return extractOrganizationNameOccurrences(text,"en"); |
| } |
| public Map<String,List<NameOccurrence>> extractOrganizationNameOccurrences(String text,String lang) { |
| return extractNameOccurrences(getNameModel("organization",lang), text,lang); |
| } |
| |
| protected Collection<String> extractNames(TokenNameFinderModel nameFinderModel, String text) { |
| return extractNameOccurrences(nameFinderModel, text, nameFinderModel.getLanguage()).keySet(); |
| } |
| |
| /** |
| * Gets/builds a TokenNameFinderModel by using {@link #openNLP} and throws |
| * {@link IllegalStateException}s in case the model could not be built or |
| * the data for the model where not found. |
| * @param the type of the named finder model |
| * @param language the language for the model |
| * @return the model or an {@link IllegalStateException} if not available |
| */ |
| private TokenNameFinderModel getNameModel(String type,String language) { |
| try { |
| TokenNameFinderModel model = openNLP.getNameModel(type, language); |
| if(model != null){ |
| return model; |
| } else { |
| throw new IllegalStateException(String.format( |
| "Unable to built Model for extracting %s from '%s' language " + |
| "texts because the model data could not be loaded.", |
| type,language)); |
| } |
| } catch (InvalidFormatException e) { |
| throw new IllegalStateException(String.format( |
| "Unable to built Model for extracting %s from '%s' language texts.", |
| type,language),e); |
| } catch (IOException e) { |
| throw new IllegalStateException(String.format( |
| "Unable to built Model for extracting %s from '%s' language texts.", |
| type,language),e); |
| } |
| } |
| /** |
| * Loads the {@link SentenceModel} for the parsed language or |
| * English as fallback if one for the language is not available |
| * @param language |
| * @return |
| */ |
| private SentenceModel getSentenceModel(String language) { |
| try { |
| SentenceModel model = openNLP.getSentenceModel(language); |
| if(model != null){ |
| return model; |
| } else { //fallback to english |
| log.info("No sentence detection modle for {}. fallback to English"); |
| model = openNLP.getSentenceModel("en"); |
| if(model == null){ |
| throw new IllegalStateException(String.format( |
| "Unable to built Model for extracting sentences neither for '%s' " + |
| "nor the fallback language 'en'.", |
| language)); |
| } else { |
| return model; |
| } |
| } |
| } catch (InvalidFormatException e) { |
| throw new IllegalStateException(String.format( |
| "Unable to built Model for extracting sentences from '%s' language texts.", |
| language),e); |
| } catch (IOException e) { |
| throw new IllegalStateException(String.format( |
| "Unable to built Model for extracting sentences from '%s' language texts.", |
| language),e); |
| } |
| } |
| /** |
| * THis method extracts NamedEntity occurrences by using existing {@link Token}s and |
| * {@link Sentence}s in the parsed {@link AnalysedText}. |
| * @param nameFinderModel the model used to find NamedEntities |
| * @param at the Analysed Text |
| * @param language the language of the text |
| * @return the found named Entity Occurrences |
| */ |
| protected Map<String,List<NameOccurrence>> extractNameOccurrences(TokenNameFinderModel nameFinderModel, |
| AnalysedText at, String language) { |
| // version with explicit sentence endings to reflect heading / paragraph |
| // structure of an HTML or PDF document converted to text |
| |
| NameFinderME finder = new NameFinderME(nameFinderModel); |
| Map<String,List<NameOccurrence>> nameOccurrences = new LinkedHashMap<String,List<NameOccurrence>>(); |
| List<Section> sentences = new ArrayList<Section>(); |
| //Holds the tokens of the previouse (pos 0) current (pos 1) and next (pos 2) sentence |
| AnalysedTextUtils.appandToList(at.getSentences(), sentences); |
| if(sentences.isEmpty()){ //no sentence annotations |
| sentences.add(at); //process as a single section |
| } |
| for (int i=0;i<sentences.size();i++) { |
| String sentence = sentences.get(i).getSpan(); |
| |
| // build a context by concatenating three sentences to be used for |
| // similarity ranking / disambiguation + contextual snippet in the |
| // extraction structure |
| List<String> contextElements = new ArrayList<String>(); |
| contextElements.add(sentence); |
| //three sentences as context |
| String context = at.getSpan().substring( |
| sentences.get(Math.max(0, i-1)).getStart(), |
| sentences.get(Math.min(sentences.size()-1, i+1)).getEnd()); |
| |
| // get the tokens, words of the current sentence |
| List<Token> tokens = new ArrayList<Token>(32); |
| List<String> words = new ArrayList<String>(32); |
| for(Iterator<Token> it =sentences.get(i).getTokens();it.hasNext();){ |
| Token t = it.next(); |
| tokens.add(t); |
| words.add(t.getSpan()); |
| } |
| Span[] nameSpans = finder.find(words.toArray(new String[words.size()])); |
| double[] probs = finder.probs(); |
| //int lastStartPosition = 0; |
| for (int j = 0; j < nameSpans.length; j++) { |
| String name = at.getSpan().substring(tokens.get(nameSpans[j].getStart()).getStart(), |
| tokens.get(nameSpans[j].getEnd()-1).getEnd()); |
| Double confidence = 1.0; |
| for (int k = nameSpans[j].getStart(); k < nameSpans[j].getEnd(); k++) { |
| confidence *= probs[k]; |
| } |
| int start = tokens.get(nameSpans[j].getStart()).getStart(); |
| int end = start + name.length(); |
| NerTag nerTag = config.getNerTag(nameSpans[j].getType()); |
| //create the occurrence for writing fise:TextAnnotations |
| NameOccurrence occurrence = new NameOccurrence(name, start, end, nerTag.getType(), |
| context, confidence); |
| List<NameOccurrence> occurrences = nameOccurrences.get(name); |
| if (occurrences == null) { |
| occurrences = new ArrayList<NameOccurrence>(); |
| } |
| occurrences.add(occurrence); |
| nameOccurrences.put(name, occurrences); |
| //add also the NerAnnotation to the AnalysedText |
| Chunk chunk = at.addChunk(start, end); |
| //TODO: build AnnotationModel based on the configured Mappings |
| chunk.addAnnotation(NER_ANNOTATION, Value.value(nerTag, confidence)); |
| } |
| } |
| finder.clearAdaptiveData(); |
| log.debug("{} name occurrences found: {}", nameOccurrences.size(), nameOccurrences); |
| return nameOccurrences; |
| } |
| |
| protected Map<String,List<NameOccurrence>> extractNameOccurrences(TokenNameFinderModel nameFinderModel, String text, String language) { |
| // version with explicit sentence endings to reflect heading / paragraph |
| // structure of an HTML or PDF document converted to text |
| String textWithDots = text.replaceAll("\\n\\n", ".\n"); |
| text = removeNonUtf8CompliantCharacters(text); |
| |
| SentenceDetectorME sentenceDetector = new SentenceDetectorME(getSentenceModel("en")); |
| |
| Span[] sentenceSpans = sentenceDetector.sentPosDetect(textWithDots); |
| |
| NameFinderME finder = new NameFinderME(nameFinderModel); |
| Tokenizer tokenizer = openNLP.getTokenizer(language); |
| Map<String,List<NameOccurrence>> nameOccurrences = new LinkedHashMap<String,List<NameOccurrence>>(); |
| for (int i = 0; i < sentenceSpans.length; i++) { |
| String sentence = sentenceSpans[i].getCoveredText(text).toString().trim(); |
| |
| // build a context by concatenating three sentences to be used for |
| // similarity ranking / disambiguation + contextual snippet in the |
| // extraction structure |
| List<String> contextElements = new ArrayList<String>(); |
| if (i > 0) { |
| CharSequence previousSentence = sentenceSpans[i - 1].getCoveredText(text); |
| contextElements.add(previousSentence.toString().trim()); |
| } |
| contextElements.add(sentence.trim()); |
| if (i + 1 < sentenceSpans.length) { |
| CharSequence nextSentence = sentenceSpans[i + 1].getCoveredText(text); |
| contextElements.add(nextSentence.toString().trim()); |
| } |
| String context = StringUtils.join(contextElements, " "); |
| |
| // extract the names in the current sentence and |
| // keep them store them with the current context |
| Span[] tokenSpans = tokenizer.tokenizePos(sentence); |
| String[] tokens = Span.spansToStrings(tokenSpans, sentence); |
| Span[] nameSpans = finder.find(tokens); |
| double[] probs = finder.probs(); |
| //int lastStartPosition = 0; |
| for (int j = 0; j < nameSpans.length; j++) { |
| String name = sentence.substring(tokenSpans[nameSpans[j].getStart()].getStart(), |
| tokenSpans[nameSpans[j].getEnd()-1].getEnd()); |
| //NOTE: With OpenNLP 1.6 the probability is now stored in the span |
| double prob = nameSpans[j].getProb(); |
| //prob == 0.0 := unspecified |
| Double confidence = prob != 0.0 ? Double.valueOf(prob) : null; |
| if(confidence == null){ //fall back to the old if it is not set. |
| for (int k = nameSpans[j].getStart(); k < nameSpans[j].getEnd(); k++) { |
| prob *= probs[k]; |
| } |
| confidence = Double.valueOf(prob); |
| } else if(confidence < 0.5d){ |
| //It looks like as if preceptron based models do return |
| //invalid probabilities. As it is expected the Named Entities |
| //with a probability < 50% are not even returned by finder.find(..) |
| //we will just ignore confidence values < 0.5 here |
| confidence = null; |
| } |
| int start = tokenSpans[nameSpans[j].getStart()].getStart(); |
| int absoluteStart = sentenceSpans[i].getStart() + start; |
| int absoluteEnd = absoluteStart + name.length(); |
| NerTag nerTag = config.getNerTag(nameSpans[j].getType()); |
| NameOccurrence occurrence = new NameOccurrence(name, absoluteStart, absoluteEnd, |
| nerTag.getType(),context, confidence); |
| |
| List<NameOccurrence> occurrences = nameOccurrences.get(name); |
| if (occurrences == null) { |
| occurrences = new ArrayList<NameOccurrence>(); |
| } |
| occurrences.add(occurrence); |
| nameOccurrences.put(name, occurrences); |
| } |
| } |
| finder.clearAdaptiveData(); |
| log.debug("{} name occurrences found: {}", nameOccurrences.size(), nameOccurrences); |
| return nameOccurrences; |
| } |
| |
| public int canEnhance(ContentItem ci) { |
| if(ContentItemHelper.getBlob(ci, SUPPORTED_MIMETYPES) != null && |
| isNerModel(extractLanguage(ci))){ |
| return ENHANCE_ASYNC; |
| } else { |
| return CANNOT_ENHANCE; |
| } |
| } |
| |
| /** |
| * Remove non UTF-8 compliant characters (typically control characters) so has to avoid polluting the |
| * annotation graph with snippets that are not serializable as XML. |
| */ |
| protected static String removeNonUtf8CompliantCharacters(final String text) { |
| if (null == text) { |
| return null; |
| } |
| StringBuilder sb = null; //initialised on the first replacement |
| for (int i = 0; i < text.length(); i++) { |
| int ch = text.codePointAt(i); |
| // remove any characters outside the valid UTF-8 range as well as all control characters |
| // except tabs and new lines |
| //NOTE: rewesten (2012-11-21) replaced the original check with the one |
| // found at http://blog.mark-mclaren.info/2007/02/invalid-xml-characters-when-valid-utf8_5873.html |
| if (!((ch == 0x9) || |
| (ch == 0xA) || |
| (ch == 0xD) || |
| ((ch >= 0x20) && (ch <= 0xD7FF)) || |
| ((ch >= 0xE000) && (ch <= 0xFFFD)) || |
| ((ch >= 0x10000) && (ch <= 0x10FFFF)))){ |
| if(sb == null){ |
| sb = new StringBuilder(text); |
| } |
| sb.setCharAt(i, ' '); |
| } |
| } |
| return sb == null ? text : sb.toString(); |
| } |
| |
| /** |
| * Extracts the language of the parsed ContentItem by using |
| * {@link EnhancementEngineHelper#getLanguage(ContentItem)} and |
| * {@link #defaultLang} as default |
| * @param ci the content item |
| * @return the language |
| */ |
| private String extractLanguage(ContentItem ci) { |
| String lang = EnhancementEngineHelper.getLanguage(ci); |
| if(lang != null){ |
| return lang; |
| } else { |
| log.info("Unable to extract language for ContentItem %s!",ci.getUri().getUnicodeString()); |
| log.info(" ... return '{}' as default",config.getDefaultLanguage()); |
| return config.getDefaultLanguage(); |
| } |
| } |
| /** |
| * This Method checks if this configuration does have a NER model for the |
| * parsed language. This checks if the pased language |
| * {@link #isProcessedLangage(String)} and any {@link #getDefaultModelTypes()} |
| * is present OR if any {@link #getSpecificNerModles(String)} is configured for the |
| * parsed language. |
| * @param lang The language to check |
| * @return if there is any NER model configured for the parsed language |
| */ |
| public boolean isNerModel(String lang){ |
| return (config.isProcessedLangage(lang) && !config.getDefaultModelTypes().isEmpty()) || |
| !config.getSpecificNerModles(lang).isEmpty(); |
| |
| } |
| } |
