| #ifndef UIMA_ANNOTATOR_TOK_H$ |
| #define UIMA_ANNOTATOR_TOK_H$ |
| /** \file annotator_tok.hpp . |
| ----------------------------------------------------------------------------- |
| |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| |
| |
| \brief Contains AnnotatorTokenizer a Unicode UIMA Tokenizer Annotator. |
| |
| -------------------------------------------------------------------------- */ |
| |
| |
| /* ----------------------------------------------------------------------- */ |
| /* Include dependencies */ |
| /* ----------------------------------------------------------------------- */ |
| |
| /* We want the timers in this annotator to be only active if the annotator specific |
| define ANNOTATOR_TIMERS is set (e.g. in the automake.pro file for this annotator) |
| in all other cases we don't want the timers. |
| Since all the timers depend on the more generic define DEBUG_TIMING we |
| define DEBUG_TIMING if and only if ANNOTATOR_TIMERS is set. |
| Specificaly we don't want the timers set when DEBUG_TIMING is defined to |
| build a generic timing driver of the whole system. |
| Our internal annotator timers would bias the whole timing driver with the |
| overhead involved in calling them in this annotator. This is why we specificaly |
| undefine DEBUG_TIMING even if it might be set in the makefile to build this |
| annotator. |
| If you want timing in this annotator use ANNOTATOR_TIMERS not DEBUG_TIMING |
| */ |
| #ifdef ANNOTATOR_TIMERS |
| # ifndef DEBUG_TIMING |
| # define DEBUG_TIMING |
| # endif |
| #else |
| # ifdef DEBUG_TIMING |
| # undef DEBUG_TIMING |
| # endif |
| #endif |
| |
| #include "uima/timedatetools.hpp" |
| #include "uima/api.hpp" /* UIMA API */ |
| ///////#include "uima/u2cpcnvrtbuff.hpp" /* U2CpConvertBuffer */ |
| #include "uima/ss_tokenizer.hpp" |
| #include "uima/internal_casimpl.hpp" |
| |
| #define STEMMER_BUF_LEN 50 |
| |
| using namespace uima; |
| |
| /** @name AnnotatorTokenizer |
| The class <TT>AnnotatorTokenizer</TT> is used a universal Unicode Tokenizer. |
| |
| It uses a little trick to check API consistency via an abstract base class, |
| without having the overhead of virtual functions in our ship version. |
| <TT>AnnotatorABase</TT> defines all non-static member functions a plug-in needs |
| to define as pure virtual functions. By making this class inherit from |
| this base class we can make sure that compilation will fail if the |
| interfaces change. |
| Since we don't really use the inheritance relationship we don't define |
| it in the ship version. |
| |
| Example: |
| \code |
| \endcode |
| @see AnnotatorABase |
| */ |
| class AnnotatorTokenizer : public Tokenizer , public TextAnnotator { |
| public: |
| /** @name Constructors */ |
| /*@{*/ |
| /** Default Constructor. |
| */ |
| AnnotatorTokenizer(void); |
| |
| /*@}*/ |
| virtual ~AnnotatorTokenizer(void); //lint !e1509: base class destructor for class 'AnnotatorABase' is not virtual |
| |
| /** @name Annotator Processing Functions */ |
| /*@{*/ |
| /** call the UIMA Annotator to initialize itself based on a UIMA engine |
| and a UIMA Configuration section and return a UIMA error code */ |
| TyErrorId |
| initialize( |
| AnnotatorContext & rclAnnotatorContext |
| ); //lint !e1909: 'virtual' assumed, see: AnnotatorABase::init(Engine &, ConfigAnnotator &) (line 79, file d:\develop\uima\current\code\engine\include\annotator_abase.hpp) |
| |
| TyErrorId typeSystemInit(TypeSystem const &); |
| |
| /** call the UIMA Annotator to deinitialize itself based on a UIMA engine |
| and return a UIMA error code */ |
| TyErrorId |
| destroy(); //lint !e1909: 'virtual' assumed, see: AnnotatorABase::deInit(Engine &) (line 83, file d:\develop\uima\current\code\engine\include\annotator_abase.hpp) |
| |
| /** call the UIMA Annotator to reconfigure itself based on a UIMA Configuration |
| section and return a UIMA error code */ |
| TyErrorId |
| reconfigure( |
| ); //lint !e1909: 'virtual' assumed, see: AnnotatorABase::config(ConfigAnnotator &) (line 87, file d:\develop\uima\current\code\engine\include\annotator_abase.hpp) |
| |
| protected: |
| /** call the UIMA Annotator to perform its doc related duty based on a UIMA engine |
| and return a UIMA error code */ |
| TyErrorId |
| process( |
| CAS &, |
| const ResultSpecification & crclTargetSet |
| ); //lint !e1909: 'virtual' assumed, see: AnnotatorABase::processDocument(Engine &, const TargetSetAT &, const TargetSetTT &) (line 91, file d:\develop\uima\current\code\engine\include\annotator_abase.hpp) |
| |
| /*@}*/ |
| protected: |
| |
| virtual int tokenCallback( unsigned long ulLocation, unsigned long ulLength, |
| TokenProperties & crclTokenProperties, |
| bool bNewPara, bool bNewSent ); |
| |
| TyDocIndex iv_uiParagraphStartIndex; |
| TyDocIndex iv_uiSentenceStartIndex; |
| // segment numbers |
| size_t iv_uiTokenNbr; |
| size_t iv_uiSentenceNbr; |
| size_t iv_uiParagraphNbr; |
| private: |
| // number of the first token/sentence/paragraph |
| const size_t iv_cuiCOUNTER_START; |
| |
| /* --------------------------------------------------*/ |
| /* config values we use */ |
| /* --------------------------------------------------*/ |
| |
| /// Enum listing all the config option we support |
| enum EnAnnotatorConfigOptions { |
| enConfigOption_TokenNumbersIncludeStopwords, |
| enConfigOption_UseRelativeTokenAndSentenceNumbers, |
| enConfigOption_IgnorePunctuationTokens, |
| // (drop inifile support) enConfigOption_CharMapConfigFilename, |
| enNumberOfConfigOptions // must be last in enum |
| }; |
| |
| // our config table |
| static const ConfigOptionInfo::StOptionInfo cv_astConfigOptionInfo[enNumberOfConfigOptions]; |
| |
| // Variables the config options are stored in: |
| |
| // if this is true the token numbers are counted including stopwords |
| bool iv_bTokenNumbersIncludeStopwords; |
| // if this is true token and sentence number are reset to 1 |
| // for each new sentence/paragraph |
| bool iv_bUseRelativeTokenAndSentenceNumbers; |
| // If true, punctuation tokens are ignored |
| bool iv_bIgnorePunctuationTokens; |
| // trace component ID |
| uima::TyComponentId iv_iTraceCompID; |
| |
| // Some pointers for quick access to UIMA objects. Initialized in init() |
| uima::internal::CASImpl * iv_pCASImpl; |
| lowlevel::FSHeap * iv_pFSHeap; |
| lowlevel::IndexRepository * iv_pIndexRepository; |
| // FSTypes and corresponding sizes |
| lowlevel::TyFSType iv_tyTokenType; |
| lowlevel::TyFeatureOffset iv_tyTokenTypeSize; |
| lowlevel::TyFSType iv_tySentenceType; |
| lowlevel::TyFeatureOffset iv_tySentenceTypeSize; |
| lowlevel::TyFSType iv_tyParagraphType; |
| lowlevel::TyFeatureOffset iv_tyParagraphTypeSize; |
| |
| // FSFeatures |
| lowlevel::TyFeatureOffset iv_tySofaFeatureOffset; |
| lowlevel::TyFeatureOffset iv_tyBeginPositionFeatureOffset; |
| lowlevel::TyFeatureOffset iv_tyEndPositionFeatureOffset; |
| |
| lowlevel::TyFeatureOffset iv_tyTokenPropertiesFeatureOffset; |
| lowlevel::TyFeatureOffset iv_tyTokenNbrFeatureOffset; |
| lowlevel::TyFeatureOffset iv_tySentenceNbrFeatureOffset; |
| lowlevel::TyFeatureOffset iv_tyParagraphNbrFeatureOffset; |
| lowlevel::TyFSFeature iv_stemFeature; |
| |
| // needed output types |
| bool iv_bIsTokenReq; |
| bool iv_bIsSentenceReq; |
| bool iv_bIsParagraphReq; |
| bool iv_stemsRequired; |
| TokenProperties iv_clTokenProperties; |
| |
| #ifdef DEBUG_TIMING |
| uima::Timer iv_clTotalTimer; |
| uima::Timer iv_clSSTokTimer; |
| uima::Timer iv_clUimaAnCreateTimer; |
| uima::Timer iv_clUimaAnSetValTimer; |
| #endif |
| |
| |
| /* --- functions --- */ |
| /* COPY CONSTRUCTOR NOT SUPPORTED */ |
| AnnotatorTokenizer(const AnnotatorTokenizer & ); //lint !e1704 |
| /* ASSIGNMENT OPERATOR NOT SUPPORTED */ |
| AnnotatorTokenizer & operator=(const AnnotatorTokenizer & crclObject); |
| |
| /// (Re-) Access config values. Used in init() and config(). |
| TyErrorId |
| getConfigValues( |
| AnnotatorContext & rclConfig |
| ); |
| |
| /// member functions for adding annotations |
| void |
| addNewTokenAnnotation( |
| TyDocIndex tyBeginPos, |
| TyDocIndex tyEndPos |
| ); |
| |
| /// member functions for adding annotations |
| void |
| addNewSentenceAnnotation( |
| TyDocIndex tyBeginPos, |
| TyDocIndex tyEndPos |
| ); |
| |
| /// member functions for adding annotations |
| void |
| addNewParagraphAnnotation( |
| TyDocIndex tyBeginPos, |
| TyDocIndex tyEndPos |
| ); |
| |
| #if defined(DEBUG_TIMING) |
| void |
| dumpTimingData( void ) const; |
| #endif |
| |
| } |
| ; /* AnnotatorTokenizer */ |
| |
| /* ----------------------------------------------------------------------- */ |
| #endif /* UIMA_ANNOTATOR_TOK_H */ |
| |
| /* <EOF> */ |
| |
