| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.lucene.analysis.icu.segmentation; |
| |
| |
| import java.io.IOException; |
| import java.io.Reader; |
| |
| import org.apache.lucene.analysis.Tokenizer; |
| import org.apache.lucene.analysis.icu.tokenattributes.ScriptAttribute; |
| import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; |
| import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; |
| import org.apache.lucene.analysis.tokenattributes.TypeAttribute; |
| import org.apache.lucene.util.AttributeFactory; |
| |
| import com.ibm.icu.lang.UCharacter; |
| import com.ibm.icu.text.BreakIterator; |
| |
| /** |
| * Breaks text into words according to UAX #29: Unicode Text Segmentation |
| * (http://www.unicode.org/reports/tr29/) |
| * <p> |
| * Words are broken across script boundaries, then segmented according to |
| * the BreakIterator and typing provided by the {@link ICUTokenizerConfig} |
| * </p> |
| * @see ICUTokenizerConfig |
| * @lucene.experimental |
| */ |
| public final class ICUTokenizer extends Tokenizer { |
| private static final int IOBUFFER = 4096; |
| private final char buffer[] = new char[IOBUFFER]; |
| /** true length of text in the buffer */ |
| private int length = 0; |
| /** length in buffer that can be evaluated safely, up to a safe end point */ |
| private int usableLength = 0; |
| /** accumulated offset of previous buffers for this reader, for offsetAtt */ |
| private int offset = 0; |
| |
| private final CompositeBreakIterator breaker; /* tokenizes a char[] of text */ |
| private final ICUTokenizerConfig config; |
| private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); |
| private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); |
| private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class); |
| private final ScriptAttribute scriptAtt = addAttribute(ScriptAttribute.class); |
| |
| /** |
| * Construct a new ICUTokenizer that breaks text into words from the given |
| * Reader. |
| * <p> |
| * The default script-specific handling is used. |
| * <p> |
| * The default attribute factory is used. |
| * |
| * @see DefaultICUTokenizerConfig |
| */ |
| public ICUTokenizer() { |
| this(new DefaultICUTokenizerConfig(true, true)); |
| } |
| |
| /** |
| * Construct a new ICUTokenizer that breaks text into words from the given |
| * Reader, using a tailored BreakIterator configuration. |
| * <p> |
| * The default attribute factory is used. |
| * |
| * @param config Tailored BreakIterator configuration |
| */ |
| public ICUTokenizer(ICUTokenizerConfig config) { |
| this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, config); |
| } |
| |
| /** |
| * Construct a new ICUTokenizer that breaks text into words from the given |
| * Reader, using a tailored BreakIterator configuration. |
| * |
| * @param factory AttributeFactory to use |
| * @param config Tailored BreakIterator configuration |
| */ |
| public ICUTokenizer(AttributeFactory factory, ICUTokenizerConfig config) { |
| super(factory); |
| this.config = config; |
| breaker = new CompositeBreakIterator(config); |
| } |
| |
| @Override |
| public boolean incrementToken() throws IOException { |
| clearAttributes(); |
| if (length == 0) |
| refill(); |
| while (!incrementTokenBuffer()) { |
| refill(); |
| if (length <= 0) // no more bytes to read; |
| return false; |
| } |
| return true; |
| } |
| |
| @Override |
| public void reset() throws IOException { |
| super.reset(); |
| breaker.setText(buffer, 0, 0); |
| length = usableLength = offset = 0; |
| } |
| |
| @Override |
| public void end() throws IOException { |
| super.end(); |
| final int finalOffset = (length < 0) ? offset : offset + length; |
| offsetAtt.setOffset(correctOffset(finalOffset), correctOffset(finalOffset)); |
| } |
| |
| /* |
| * This tokenizes text based upon the longest matching rule, and because of |
| * this, isn't friendly to a Reader. |
| * |
| * Text is read from the input stream in 4kB chunks. Within a 4kB chunk of |
| * text, the last unambiguous break point is found (in this implementation: |
| * white space character) Any remaining characters represent possible partial |
| * words, so are appended to the front of the next chunk. |
| * |
| * There is the possibility that there are no unambiguous break points within |
| * an entire 4kB chunk of text (binary data). So there is a maximum word limit |
| * of 4kB since it will not try to grow the buffer in this case. |
| */ |
| |
| /** |
| * Returns the last unambiguous break position in the text. |
| * |
| * @return position of character, or -1 if one does not exist |
| */ |
| private int findSafeEnd() { |
| for (int i = length - 1; i >= 0; i--) |
| if (UCharacter.isWhitespace(buffer[i])) |
| return i + 1; |
| return -1; |
| } |
| |
| /** |
| * Refill the buffer, accumulating the offset and setting usableLength to the |
| * last unambiguous break position |
| * |
| * @throws IOException If there is a low-level I/O error. |
| */ |
| private void refill() throws IOException { |
| offset += usableLength; |
| int leftover = length - usableLength; |
| System.arraycopy(buffer, usableLength, buffer, 0, leftover); |
| int requested = buffer.length - leftover; |
| int returned = read(input, buffer, leftover, requested); |
| length = returned + leftover; |
| if (returned < requested) /* reader has been emptied, process the rest */ |
| usableLength = length; |
| else { /* still more data to be read, find a safe-stopping place */ |
| usableLength = findSafeEnd(); |
| if (usableLength < 0) |
| usableLength = length; /* |
| * more than IOBUFFER of text without space, |
| * gonna possibly truncate tokens |
| */ |
| } |
| |
| breaker.setText(buffer, 0, Math.max(0, usableLength)); |
| } |
| |
| // TODO: refactor to a shared readFully somewhere |
| // (NGramTokenizer does this too): |
| /** commons-io's readFully, but without bugs if offset != 0 */ |
| private static int read(Reader input, char[] buffer, int offset, int length) throws IOException { |
| assert length >= 0 : "length must not be negative: " + length; |
| |
| int remaining = length; |
| while ( remaining > 0 ) { |
| int location = length - remaining; |
| int count = input.read( buffer, offset + location, remaining ); |
| if ( -1 == count ) { // EOF |
| break; |
| } |
| remaining -= count; |
| } |
| return length - remaining; |
| } |
| |
| /* |
| * return true if there is a token from the buffer, or null if it is |
| * exhausted. |
| */ |
| private boolean incrementTokenBuffer() { |
| int start = breaker.current(); |
| if (start == BreakIterator.DONE) |
| return false; // BreakIterator exhausted |
| |
| // find the next set of boundaries, skipping over non-tokens (rule status 0) |
| int end = breaker.next(); |
| while (start != BreakIterator.DONE && breaker.getRuleStatus() == 0) { |
| start = end; |
| end = breaker.next(); |
| } |
| |
| if (start == BreakIterator.DONE) |
| return false; // BreakIterator exhausted |
| |
| termAtt.copyBuffer(buffer, start, end - start); |
| offsetAtt.setOffset(correctOffset(offset + start), correctOffset(offset + end)); |
| typeAtt.setType(config.getType(breaker.getScriptCode(), breaker.getRuleStatus())); |
| scriptAtt.setCode(breaker.getScriptCode()); |
| |
| return true; |
| } |
| } |