| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.manifoldcf.agents.transformation.opennlp; |
| |
| import java.io.*; |
| |
| import java.nio.charset.StandardCharsets; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.HashMap; |
| import java.util.List; |
| import java.util.Locale; |
| import java.util.Map; |
| import java.util.HashMap; |
| import java.util.TreeMap; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| import java.util.HashSet; |
| |
| import opennlp.tools.namefind.NameFinderME; |
| import opennlp.tools.sentdetect.SentenceDetector; |
| import opennlp.tools.tokenize.Tokenizer; |
| import opennlp.tools.util.Span; |
| |
| import org.apache.commons.io.IOUtils; |
| import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity; |
| import org.apache.manifoldcf.agents.interfaces.IOutputCheckActivity; |
| import org.apache.manifoldcf.agents.interfaces.RepositoryDocument; |
| import org.apache.manifoldcf.agents.interfaces.ServiceInterruption; |
| import org.apache.manifoldcf.agents.system.Logging; |
| import org.apache.manifoldcf.agents.system.ManifoldCF; |
| import org.apache.manifoldcf.agents.transformation.BaseTransformationConnector; |
| import org.apache.manifoldcf.core.interfaces.IHTTPOutput; |
| import org.apache.manifoldcf.core.interfaces.IPostParameters; |
| import org.apache.manifoldcf.core.interfaces.ManifoldCFException; |
| import org.apache.manifoldcf.core.interfaces.Specification; |
| import org.apache.manifoldcf.core.interfaces.SpecificationNode; |
| import org.apache.manifoldcf.core.interfaces.VersionContext; |
| |
| public class OpenNlpExtractor extends BaseTransformationConnector { |
| private static final String EDIT_SPECIFICATION_JS = "editSpecification.js"; |
| private static final String EDIT_SPECIFICATION_OPENNLP_HTML = "editSpecification_OpenNLP.html"; |
| private static final String VIEW_SPECIFICATION_HTML = "viewSpecification.html"; |
| |
| protected static int maximumExtractionCharacters = 524288; |
| |
| // Meta-data fields added by this connector |
| private static final String PERSONS = "ner_people"; |
| private static final String LOCATIONS = "ner_locations"; |
| private static final String ORGANIZATIONS = "ner_organizations"; |
| |
| protected static final String ACTIVITY_EXTRACT = "extract"; |
| |
| protected static final String[] activitiesList = new String[] { ACTIVITY_EXTRACT }; |
| |
| protected final File fileDirectory = ManifoldCF.getFileProperty(ManifoldCF.fileResourcesProperty); |
| |
| /** We handle up to 64K in memory; after that we go to disk. */ |
| protected static final long inMemoryMaximumFile = 65536; |
| |
| |
| /** |
| * Return a list of activities that this connector generates. The connector |
| * does NOT need to be connected before this method is called. |
| * |
| * @return the set of activities. |
| */ |
| @Override |
| public String[] getActivitiesList() { |
| return activitiesList; |
| } |
| |
| /** |
| * Get a pipeline version string, given a pipeline specification object. The |
| * version string is used to uniquely describe the pertinent details of the |
| * specification and the configuration, to allow the Connector Framework to |
| * determine whether a document will need to be processed again. Note that |
| * the contents of any document cannot be considered by this method; only |
| * configuration and specification information can be considered. |
| * |
| * This method presumes that the underlying connector object has been |
| * configured. |
| * |
| * @param spec |
| * is the current pipeline specification object for this |
| * connection for the job that is doing the crawling. |
| * @return a string, of unlimited length, which uniquely describes |
| * configuration and specification in such a way that if two such |
| * strings are equal, nothing that affects how or whether the |
| * document is indexed will be different. |
| */ |
| @Override |
| public VersionContext getPipelineDescription(Specification os) throws ManifoldCFException, ServiceInterruption { |
| SpecPacker sp = new SpecPacker(os); |
| return new VersionContext(sp.toPackedString(), params, os); |
| } |
| |
| /** |
| * Add (or replace) a document in the output data store using the connector. |
| * This method presumes that the connector object has been configured, and |
| * it is thus able to communicate with the output data store should that be |
| * necessary. The OutputSpecification is *not* provided to this method, |
| * because the goal is consistency, and if output is done it must be |
| * consistent with the output description, since that was what was partly |
| * used to determine if output should be taking place. So it may be |
| * necessary for this method to decode an output description string in order |
| * to determine what should be done. |
| * |
| * @param documentURI |
| * is the URI of the document. The URI is presumed to be the |
| * unique identifier which the output data store will use to |
| * process and serve the document. This URI is constructed by the |
| * repository connector which fetches the document, and is thus |
| * universal across all output connectors. |
| * @param outputDescription |
| * is the description string that was constructed for this |
| * document by the getOutputDescription() method. |
| * @param document |
| * is the document data to be processed (handed to the output |
| * data store). |
| * @param authorityNameString |
| * is the name of the authority responsible for authorizing any |
| * access tokens passed in with the repository document. May be |
| * null. |
| * @param activities |
| * is the handle to an object that the implementer of a pipeline |
| * connector may use to perform operations, such as logging |
| * processing activity, or sending a modified document to the |
| * next stage in the pipeline. |
| * @return the document status (accepted or permanently rejected). |
| * @throws IOException |
| * only if there's a stream error reading the document data. |
| */ |
| @Override |
| public int addOrReplaceDocumentWithException(String documentURI, VersionContext pipelineDescription, |
| RepositoryDocument document, String authorityNameString, IOutputAddActivity activities) |
| throws ManifoldCFException, ServiceInterruption, IOException { |
| // assumes use of Tika extractor before using this connector |
| Logging.agents.debug("Starting OpenNlp extraction"); |
| |
| SpecPacker sp = new SpecPacker(pipelineDescription.getSpecification()); |
| |
| // In order to be able to replay the input stream both for extraction and for downstream use, |
| // we need to page through it, some number of characters at a time, and write those into a local buffer. |
| // We can do this at the same time we're extracting, if we're clever. |
| |
| // Set up to spool back the original content, using either memory or disk, whichever makes sense. |
| DestinationStorage ds; |
| if (document.getBinaryLength() <= inMemoryMaximumFile) { |
| ds = new MemoryDestinationStorage((int)document.getBinaryLength()); |
| } else { |
| ds = new FileDestinationStorage(); |
| } |
| |
| try { |
| |
| // For logging, we'll need all of this |
| long startTime = System.currentTimeMillis(); |
| String resultCode = "OK"; |
| String description = null; |
| Long length = null; |
| |
| final MetadataAccumulator ma = new MetadataAccumulator(sp, document.getBinaryLength()); |
| |
| try { |
| |
| // Page through document content, saving it aside into destination storage, while also extracting the content |
| final OutputStream os = ds.getOutputStream(); |
| try { |
| // We presume that the content is utf-8!! Thus it has to have been run through the TikaExtractor, or equivalent. |
| // |
| // We're going to be paging through the input stream by chunks of characters. Each chunk will then be passed to the |
| // output stream (os) via a writer, as well as to the actual code that invokes the nlp sentence extraction. |
| |
| // We need an output writer that converts the input into characters. |
| // |
| Writer w = new OutputStreamWriter(os, "utf-8"); |
| try { |
| Reader r = new InputStreamReader(document.getBinaryStream(), "utf-8"); |
| try { |
| // Now, page through! |
| // It's too bad we have to convert FROM utf-8 and then back TO utf-8, but that can't be helped. |
| char[] characterBuffer = new char[65536]; |
| while (true) { |
| int amt = r.read(characterBuffer); |
| if (amt == -1) { |
| break; |
| } |
| // Write into the copy buffer |
| w.write(characterBuffer,0,amt); |
| // Also do the processing |
| ma.acceptCharacters(characterBuffer,amt); |
| } |
| // Do not close the reader; the underlying stream will be closed by our caller when the RepositoryDocument is done with |
| } catch (IOException e) { |
| // These are errors from reading the RepositoryDocument input stream; we handle them accordingly. |
| resultCode = e.getClass().getSimpleName().toUpperCase(Locale.ROOT); |
| description = e.getMessage(); |
| throw e; |
| } |
| } finally { |
| w.flush(); |
| } |
| } |
| finally |
| { |
| os.close(); |
| length = new Long(ds.getBinaryLength()); |
| } |
| |
| } |
| finally |
| { |
| // Log the extraction processing |
| activities.recordActivity(new Long(startTime), ACTIVITY_EXTRACT, length, documentURI, |
| resultCode, description); |
| } |
| |
| ma.done(); |
| |
| // Parsing complete! |
| // Create a copy of Repository Document |
| RepositoryDocument docCopy = document.duplicate(); |
| |
| // Get new stream length |
| long newBinaryLength = ds.getBinaryLength(); |
| // Open new input stream |
| InputStream is = ds.getInputStream(); |
| try |
| { |
| docCopy.setBinary(is,newBinaryLength); |
| |
| // add named entity meta-data |
| Map<String,Set<String>> nerMap = ma.getMetadata(); |
| if (!nerMap.isEmpty()) { |
| for (Entry<String, Set<String>> entry : nerMap.entrySet()) { |
| Set<String> neList = entry.getValue(); |
| String[] neArray = neList.toArray(new String[0]); |
| docCopy.addField(entry.getKey(), neArray); |
| } |
| } |
| |
| // Send new document downstream |
| return activities.sendDocument(documentURI,docCopy); |
| } finally { |
| is.close(); |
| } |
| } finally { |
| ds.close(); |
| } |
| } |
| |
| private final static Set<String> acceptableMimeTypes = new HashSet<String>(); |
| static |
| { |
| acceptableMimeTypes.add("text/plain;charset=utf-8"); |
| acceptableMimeTypes.add("text/plain;charset=ascii"); |
| acceptableMimeTypes.add("text/plain;charset=us-ascii"); |
| acceptableMimeTypes.add("text/plain"); |
| } |
| |
| /** Detect if a mime type is acceptable or not. This method is used to determine whether it makes sense to fetch a document |
| * in the first place. |
| *@param pipelineDescription is the document's pipeline version string, for this connection. |
| *@param mimeType is the mime type of the document. |
| *@param checkActivity is an object including the activities that can be performed by this method. |
| *@return true if the mime type can be accepted by this connector. |
| */ |
| @Override |
| public boolean checkMimeTypeIndexable(VersionContext pipelineDescription, String mimeType, IOutputCheckActivity checkActivity) |
| throws ManifoldCFException, ServiceInterruption |
| { |
| if (mimeType == null || !acceptableMimeTypes.contains(mimeType.toLowerCase(Locale.ROOT))) { |
| return false; |
| } |
| // Do a downstream check too |
| return super.checkMimeTypeIndexable(pipelineDescription, mimeType, checkActivity); |
| } |
| |
| // //////////////////////// |
| // UI Methods |
| // //////////////////////// |
| |
| /** |
| * Obtain the name of the form check javascript method to call. |
| * |
| * @param connectionSequenceNumber |
| * is the unique number of this connection within the job. |
| * @return the name of the form check javascript method. |
| */ |
| @Override |
| public String getFormCheckJavascriptMethodName(int connectionSequenceNumber) { |
| return "s" + connectionSequenceNumber + "_checkSpecification"; |
| } |
| |
| /** |
| * Obtain the name of the form presave check javascript method to call. |
| * |
| * @param connectionSequenceNumber |
| * is the unique number of this connection within the job. |
| * @return the name of the form presave check javascript method. |
| */ |
| @Override |
| public String getFormPresaveCheckJavascriptMethodName(int connectionSequenceNumber) { |
| return "s" + connectionSequenceNumber + "_checkSpecificationForSave"; |
| } |
| |
| /** |
| * Output the specification header section. This method is called in the |
| * head section of a job page which has selected an output connection of the |
| * current type. Its purpose is to add the required tabs to the list, and to |
| * output any javascript methods that might be needed by the job editing |
| * HTML. |
| * |
| * @param out |
| * is the output to which any HTML should be sent. |
| * @param locale |
| * is the preferred local of the output. |
| * @param os |
| * is the current output specification for this job. |
| * @param connectionSequenceNumber |
| * is the unique number of this connection within the job. |
| * @param tabsArray |
| * is an array of tab names. Add to this array any tab names that |
| * are specific to the connector. |
| */ |
| @Override |
| public void outputSpecificationHeader(IHTTPOutput out, Locale locale, Specification os, |
| int connectionSequenceNumber, List<String> tabsArray) throws ManifoldCFException, IOException { |
| Map<String, Object> paramMap = new HashMap<String, Object>(); |
| paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber)); |
| |
| tabsArray.add(Messages.getString(locale, "OpenNlpExtractor.OpenNLPTabName")); |
| |
| Messages.outputResourceWithVelocity(out, locale, EDIT_SPECIFICATION_JS, paramMap); |
| } |
| |
| /** |
| * Output the specification body section. This method is called in the body |
| * section of a job page which has selected an output connection of the |
| * current type. Its purpose is to present the required form elements for |
| * editing. The coder can presume that the HTML that is output from this |
| * configuration will be within appropriate <html>, <body>, and <form> tags. |
| * The name of the form is "editjob". |
| * |
| * @param out |
| * is the output to which any HTML should be sent. |
| * @param locale |
| * is the preferred local of the output. |
| * @param os |
| * is the current output specification for this job. |
| * @param connectionSequenceNumber |
| * is the unique number of this connection within the job. |
| * @param actualSequenceNumber |
| * is the connection within the job that has currently been |
| * selected. |
| * @param tabName |
| * is the current tab name. |
| */ |
| @Override |
| public void outputSpecificationBody(IHTTPOutput out, Locale locale, Specification os, int connectionSequenceNumber, |
| int actualSequenceNumber, String tabName) throws ManifoldCFException, IOException { |
| Map<String, Object> paramMap = new HashMap<String, Object>(); |
| |
| paramMap.put("TABNAME", tabName); |
| paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber)); |
| paramMap.put("SELECTEDNUM", Integer.toString(actualSequenceNumber)); |
| |
| fillInOpenNLPSpecificationMap(paramMap, os); |
| setUpOpenNLPSpecificationMap(paramMap); |
| |
| Messages.outputResourceWithVelocity(out, locale, EDIT_SPECIFICATION_OPENNLP_HTML, paramMap); |
| } |
| |
| /** |
| * Process a specification post. This method is called at the start of job's |
| * edit or view page, whenever there is a possibility that form data for a |
| * connection has been posted. Its purpose is to gather form information and |
| * modify the output specification accordingly. The name of the posted form |
| * is "editjob". |
| * |
| * @param variableContext |
| * contains the post data, including binary file-upload |
| * information. |
| * @param locale |
| * is the preferred local of the output. |
| * @param os |
| * is the current output specification for this job. |
| * @param connectionSequenceNumber |
| * is the unique number of this connection within the job. |
| * @return null if all is well, or a string error message if there is an |
| * error that should prevent saving of the job (and cause a |
| * redirection to an error page). |
| */ |
| @Override |
| public String processSpecificationPost(IPostParameters variableContext, Locale locale, Specification os, |
| int connectionSequenceNumber) throws ManifoldCFException { |
| String seqPrefix = "s" + connectionSequenceNumber + "_"; |
| |
| SpecificationNode node = new SpecificationNode(OpenNlpExtractorConfig.NODE_SMODEL_PATH); |
| String smodelPath = variableContext.getParameter(seqPrefix + "smodelpath"); |
| if (smodelPath != null) { |
| node.setAttribute(OpenNlpExtractorConfig.ATTRIBUTE_VALUE, smodelPath); |
| } else { |
| node.setAttribute(OpenNlpExtractorConfig.ATTRIBUTE_VALUE, ""); |
| } |
| os.addChild(os.getChildCount(), node); |
| |
| node = new SpecificationNode(OpenNlpExtractorConfig.NODE_TMODEL_PATH); |
| String tmodelPath = variableContext.getParameter(seqPrefix + "tmodelpath"); |
| if (tmodelPath != null) { |
| node.setAttribute(OpenNlpExtractorConfig.ATTRIBUTE_VALUE, tmodelPath); |
| } else { |
| node.setAttribute(OpenNlpExtractorConfig.ATTRIBUTE_VALUE, ""); |
| } |
| os.addChild(os.getChildCount(), node); |
| |
| String modelCount = variableContext.getParameter(seqPrefix+"model_count"); |
| if (modelCount != null) |
| { |
| int count = Integer.parseInt(modelCount); |
| // Delete old spec data, including legacy node types we no longer use |
| int i = 0; |
| while (i < os.getChildCount()) |
| { |
| SpecificationNode cn = os.getChild(i); |
| if (cn.getType().equals(OpenNlpExtractorConfig.NODE_FINDERMODEL)) |
| os.removeChild(i); |
| else |
| i++; |
| } |
| |
| // Now, go through form data |
| for (int j = 0; j < count; j++) |
| { |
| String op = variableContext.getParameter(seqPrefix+"model_"+j+"_op"); |
| if (op != null && op.equals("Delete")) |
| continue; |
| String paramName = variableContext.getParameter(seqPrefix+"model_"+j+"_parametername"); |
| String modelFile = variableContext.getParameter(seqPrefix+"model_"+j+"_modelfile"); |
| SpecificationNode sn = new SpecificationNode(OpenNlpExtractorConfig.NODE_FINDERMODEL); |
| sn.setAttribute(OpenNlpExtractorConfig.ATTRIBUTE_PARAMETERNAME,paramName); |
| sn.setAttribute(OpenNlpExtractorConfig.ATTRIBUTE_MODELFILE,modelFile); |
| os.addChild(os.getChildCount(),sn); |
| } |
| // Look for add operation |
| String addOp = variableContext.getParameter(seqPrefix+"model_op"); |
| if (addOp != null && addOp.equals("Add")) |
| { |
| String paramName = variableContext.getParameter(seqPrefix+"model_parametername"); |
| String modelFile = variableContext.getParameter(seqPrefix+"model_modelfile"); |
| SpecificationNode sn = new SpecificationNode(OpenNlpExtractorConfig.NODE_FINDERMODEL); |
| sn.setAttribute(OpenNlpExtractorConfig.ATTRIBUTE_PARAMETERNAME,paramName); |
| sn.setAttribute(OpenNlpExtractorConfig.ATTRIBUTE_MODELFILE,modelFile); |
| os.addChild(os.getChildCount(),sn); |
| } |
| |
| } |
| |
| return null; |
| } |
| |
| /** |
| * View specification. This method is called in the body section of a job's |
| * view page. Its purpose is to present the output specification information |
| * to the user. The coder can presume that the HTML that is output from this |
| * configuration will be within appropriate <html> and <body> tags. |
| * |
| * @param out |
| * is the output to which any HTML should be sent. |
| * @param locale |
| * is the preferred local of the output. |
| * @param connectionSequenceNumber |
| * is the unique number of this connection within the job. |
| * @param os |
| * is the current output specification for this job. |
| */ |
| @Override |
| public void viewSpecification(IHTTPOutput out, Locale locale, Specification os, int connectionSequenceNumber) |
| throws ManifoldCFException, IOException { |
| Map<String, Object> paramMap = new HashMap<String, Object>(); |
| paramMap.put("SEQNUM", Integer.toString(connectionSequenceNumber)); |
| |
| fillInOpenNLPSpecificationMap(paramMap, os); |
| Messages.outputResourceWithVelocity(out, locale, VIEW_SPECIFICATION_HTML, paramMap); |
| } |
| |
| protected void setUpOpenNLPSpecificationMap(Map<String, Object> paramMap) |
| throws ManifoldCFException { |
| final String[] fileNames = getModelList(); |
| paramMap.put("FILENAMES", fileNames); |
| } |
| |
| protected static void fillInOpenNLPSpecificationMap(Map<String, Object> paramMap, Specification os) { |
| String sModelPath = ""; |
| String tModelPath = ""; |
| final List<Map<String,String>> finderModels = new ArrayList<>(); |
| |
| for (int i = 0; i < os.getChildCount(); i++) { |
| SpecificationNode sn = os.getChild(i); |
| if (sn.getType().equals(OpenNlpExtractorConfig.NODE_SMODEL_PATH)) { |
| sModelPath = sn.getAttributeValue(OpenNlpExtractorConfig.ATTRIBUTE_VALUE); |
| if (sModelPath == null) { |
| sModelPath = ""; |
| } |
| } else if (sn.getType().equals(OpenNlpExtractorConfig.NODE_TMODEL_PATH)) { |
| tModelPath = sn.getAttributeValue(OpenNlpExtractorConfig.ATTRIBUTE_VALUE); |
| if (tModelPath == null) { |
| tModelPath = ""; |
| } |
| } else if (sn.getType().equals(OpenNlpExtractorConfig.NODE_FINDERMODEL)) { |
| final String parameterName = sn.getAttributeValue(OpenNlpExtractorConfig.ATTRIBUTE_PARAMETERNAME); |
| final String modelFile = sn.getAttributeValue(OpenNlpExtractorConfig.ATTRIBUTE_MODELFILE); |
| final Map<String,String> modelRecord = new HashMap<>(); |
| modelRecord.put("parametername", parameterName); |
| modelRecord.put("modelfile", modelFile); |
| finderModels.add(modelRecord); |
| } |
| |
| } |
| paramMap.put("SMODELPATH", sModelPath); |
| paramMap.put("TMODELPATH", tModelPath); |
| paramMap.put("MODELS", finderModels); |
| } |
| |
| protected static int handleIOException(IOException e) |
| throws ManifoldCFException |
| { |
| // IOException reading from our local storage... |
| if (e instanceof InterruptedIOException) |
| throw new ManifoldCFException(e.getMessage(),e,ManifoldCFException.INTERRUPTED); |
| throw new ManifoldCFException(e.getMessage(),e); |
| } |
| |
| protected String[] getModelList() throws ManifoldCFException { |
| if (fileDirectory == null) { |
| return new String[0]; |
| } |
| final String[] files = fileDirectory.list(new FileFilter()); |
| // Sort it!! |
| java.util.Arrays.sort(files); |
| return files; |
| } |
| |
| protected static class FileFilter implements FilenameFilter { |
| @Override |
| public boolean accept(final File dir, final String name) { |
| return new File(dir, name).isFile(); |
| } |
| } |
| |
| /** An instance of this class receives characters in 64K chunks, and needs to accumulate |
| * extracted metadata that this transformer will pass down. |
| */ |
| protected class MetadataAccumulator { |
| |
| char[] characterBuffer = null; |
| int bufferPointer = 0; |
| |
| final int bufferSize; |
| |
| final SentenceDetector sentenceDetector; |
| final Tokenizer tokenizer; |
| final Map<String,NameFinderME> finders = new HashMap<>(); |
| final Map<String,Set<String>> tokenLists = new HashMap<>(); |
| |
| public MetadataAccumulator(final SpecPacker sp, |
| final long bytesize) |
| throws ManifoldCFException { |
| try { |
| sentenceDetector = OpenNlpExtractorConfig.sentenceDetector(new File(fileDirectory,sp.getSModelPath())); |
| tokenizer = OpenNlpExtractorConfig.tokenizer(new File(fileDirectory,sp.getTModelPath())); |
| final Map<String,String> finderFiles = sp.getFinderModels(); |
| for (String paramName : finderFiles.keySet()) { |
| finders.put(paramName, OpenNlpExtractorConfig.finder(new File(fileDirectory,finderFiles.get(paramName)))); |
| } |
| } catch (IOException e) { |
| throw new ManifoldCFException(e.getMessage(), e); |
| } |
| if (bytesize > maximumExtractionCharacters) { |
| bufferSize = maximumExtractionCharacters; |
| } else { |
| bufferSize = (int)bytesize; |
| } |
| } |
| |
| /** Accept characters, including actual count. |
| */ |
| public void acceptCharacters(final char[] buffer, int amt) { |
| if (characterBuffer == null) { |
| characterBuffer = new char[bufferSize]; |
| } |
| int copyAmt; |
| if (amt > bufferSize - bufferPointer) { |
| copyAmt = bufferSize - bufferPointer; |
| } else { |
| copyAmt = amt; |
| } |
| int sourcePtr = 0; |
| while (copyAmt > 0) { |
| characterBuffer[bufferPointer++] = buffer[sourcePtr++]; |
| copyAmt--; |
| } |
| } |
| |
| public void done() { |
| if (bufferPointer == 0 || characterBuffer == null) { |
| return; |
| } |
| |
| // Make a string freom the character array |
| final String textContent = new String(characterBuffer, 0, bufferPointer); |
| |
| // Break into sentences, tokens, and then people, locations, and organizations |
| String[] sentences = sentenceDetector.sentDetect(textContent); |
| for (String sentence : sentences) { |
| String[] tokens = tokenizer.tokenize(sentence); |
| |
| for (String parameterName : finders.keySet()) { |
| Set<String> stringSet = tokenLists.get(parameterName); |
| if (stringSet == null) { |
| stringSet = new HashSet<String>(); |
| tokenLists.put(parameterName, stringSet); |
| } |
| |
| Span[] spans = finders.get(parameterName).find(tokens); |
| stringSet.addAll(Arrays.asList(Span.spansToStrings(spans, tokens))); |
| } |
| } |
| } |
| |
| public Map<String,Set<String>> getMetadata() { |
| return tokenLists; |
| } |
| |
| } |
| |
| protected static interface DestinationStorage { |
| /** Get the output stream to write to. Caller should explicitly close this stream when done writing. |
| */ |
| public OutputStream getOutputStream() |
| throws ManifoldCFException; |
| |
| /** Get new binary length. |
| */ |
| public long getBinaryLength() |
| throws ManifoldCFException; |
| |
| /** Get the input stream to read from. Caller should explicitly close this stream when done reading. |
| */ |
| public InputStream getInputStream() |
| throws ManifoldCFException; |
| |
| /** Close the object and clean up everything. |
| * This should be called when the data is no longer needed. |
| */ |
| public void close() |
| throws ManifoldCFException; |
| } |
| |
| protected static class FileDestinationStorage implements DestinationStorage { |
| protected final File outputFile; |
| protected final OutputStream outputStream; |
| |
| public FileDestinationStorage() |
| throws ManifoldCFException |
| { |
| File outputFile; |
| OutputStream outputStream; |
| try |
| { |
| outputFile = File.createTempFile("mcftika","tmp"); |
| outputStream = new FileOutputStream(outputFile); |
| } |
| catch (IOException e) |
| { |
| handleIOException(e); |
| outputFile = null; |
| outputStream = null; |
| } |
| this.outputFile = outputFile; |
| this.outputStream = outputStream; |
| } |
| |
| @Override |
| public OutputStream getOutputStream() |
| throws ManifoldCFException |
| { |
| return outputStream; |
| } |
| |
| /** Get new binary length. |
| */ |
| @Override |
| public long getBinaryLength() |
| throws ManifoldCFException |
| { |
| return outputFile.length(); |
| } |
| |
| /** Get the input stream to read from. Caller should explicitly close this stream when done reading. |
| */ |
| @Override |
| public InputStream getInputStream() |
| throws ManifoldCFException |
| { |
| try |
| { |
| return new FileInputStream(outputFile); |
| } |
| catch (IOException e) |
| { |
| handleIOException(e); |
| return null; |
| } |
| } |
| |
| /** Close the object and clean up everything. |
| * This should be called when the data is no longer needed. |
| */ |
| @Override |
| public void close() |
| throws ManifoldCFException |
| { |
| outputFile.delete(); |
| } |
| |
| } |
| |
| protected static class MemoryDestinationStorage implements DestinationStorage { |
| protected final ByteArrayOutputStream outputStream; |
| |
| public MemoryDestinationStorage(int sizeHint) |
| { |
| outputStream = new ByteArrayOutputStream(sizeHint); |
| } |
| |
| @Override |
| public OutputStream getOutputStream() |
| throws ManifoldCFException |
| { |
| return outputStream; |
| } |
| |
| /** Get new binary length. |
| */ |
| @Override |
| public long getBinaryLength() |
| throws ManifoldCFException |
| { |
| return outputStream.size(); |
| } |
| |
| /** Get the input stream to read from. Caller should explicitly close this stream when done reading. |
| */ |
| @Override |
| public InputStream getInputStream() |
| throws ManifoldCFException |
| { |
| return new ByteArrayInputStream(outputStream.toByteArray()); |
| } |
| |
| /** Close the object and clean up everything. |
| * This should be called when the data is no longer needed. |
| */ |
| public void close() |
| throws ManifoldCFException |
| { |
| } |
| |
| } |
| |
| protected static class SpecPacker { |
| |
| private final String sModelPath; |
| private final String tModelPath; |
| private final Map<String, String> models = new TreeMap<>(); |
| |
| public SpecPacker(Specification os) { |
| String sModelPath = null; |
| String tModelPath = null; |
| |
| for (int i = 0; i < os.getChildCount(); i++) { |
| SpecificationNode sn = os.getChild(i); |
| |
| if (sn.getType().equals(OpenNlpExtractorConfig.NODE_SMODEL_PATH)) { |
| sModelPath = sn.getAttributeValue(OpenNlpExtractorConfig.ATTRIBUTE_VALUE); |
| } |
| if (sn.getType().equals(OpenNlpExtractorConfig.NODE_TMODEL_PATH)) { |
| tModelPath = sn.getAttributeValue(OpenNlpExtractorConfig.ATTRIBUTE_VALUE); |
| } |
| if (sn.getType().equals(OpenNlpExtractorConfig.NODE_FINDERMODEL)) { |
| final String parameterName = sn.getAttributeValue(OpenNlpExtractorConfig.ATTRIBUTE_PARAMETERNAME); |
| final String modelFile = sn.getAttributeValue(OpenNlpExtractorConfig.ATTRIBUTE_MODELFILE); |
| models.put(parameterName, modelFile); |
| } |
| |
| } |
| this.sModelPath = sModelPath; |
| this.tModelPath = tModelPath; |
| } |
| |
| public String toPackedString() { |
| StringBuilder sb = new StringBuilder(); |
| |
| // extract nouns |
| if (sModelPath != null) |
| sb.append(sModelPath); |
| sb.append(","); |
| if (tModelPath != null) |
| sb.append(tModelPath); |
| sb.append("["); |
| for (String parameterName : models.keySet()) { |
| sb.append(parameterName).append("=").append(models.get(parameterName)).append(","); |
| } |
| |
| return sb.toString(); |
| } |
| |
| public String getSModelPath() { |
| return sModelPath; |
| } |
| |
| public String getTModelPath() { |
| return tModelPath; |
| } |
| |
| public Map<String, String> getFinderModels() { |
| return models; |
| } |
| |
| } |
| |
| } |
