| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.lucene.analysis.in; |
| |
| |
| import java.util.BitSet; |
| import java.util.IdentityHashMap; |
| import static java.lang.Character.UnicodeBlock.*; |
| import static org.apache.lucene.analysis.util.StemmerUtil.*; |
| |
| /** |
| * Normalizes the Unicode representation of text in Indian languages. |
| * <p> |
| * Follows guidelines from Unicode 5.2, chapter 6, South Asian Scripts I |
| * and graphical decompositions from http://ldc.upenn.edu/myl/IndianScriptsUnicode.html |
| * </p> |
| */ |
| public class IndicNormalizer { |
| |
| private static class ScriptData { |
| final int flag; |
| final int base; |
| BitSet decompMask; |
| |
| ScriptData(int flag, int base) { |
| this.flag = flag; |
| this.base = base; |
| } |
| } |
| |
| private static final IdentityHashMap<Character.UnicodeBlock,ScriptData> scripts = |
| new IdentityHashMap<>(9); |
| |
| private static int flag(Character.UnicodeBlock ub) { |
| return scripts.get(ub).flag; |
| } |
| |
| static { |
| scripts.put(DEVANAGARI, new ScriptData(1, 0x0900)); |
| scripts.put(BENGALI, new ScriptData(2, 0x0980)); |
| scripts.put(GURMUKHI, new ScriptData(4, 0x0A00)); |
| scripts.put(GUJARATI, new ScriptData(8, 0x0A80)); |
| scripts.put(ORIYA, new ScriptData(16, 0x0B00)); |
| scripts.put(TAMIL, new ScriptData(32, 0x0B80)); |
| scripts.put(TELUGU, new ScriptData(64, 0x0C00)); |
| scripts.put(KANNADA, new ScriptData(128, 0x0C80)); |
| scripts.put(MALAYALAM, new ScriptData(256, 0x0D00)); |
| } |
| |
| /** |
| * Decompositions according to Unicode 5.2, |
| * and http://ldc.upenn.edu/myl/IndianScriptsUnicode.html |
| * |
| * Most of these are not handled by unicode normalization anyway. |
| * |
| * The numbers here represent offsets into the respective codepages, |
| * with -1 representing null and 0xFF representing zero-width joiner. |
| * |
| * the columns are: ch1, ch2, ch3, res, flags |
| * ch1, ch2, and ch3 are the decomposition |
| * res is the composition, and flags are the scripts to which it applies. |
| */ |
| private static final int decompositions[][] = { |
| /* devanagari, gujarati vowel candra O */ |
| { 0x05, 0x3E, 0x45, 0x11, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari short O */ |
| { 0x05, 0x3E, 0x46, 0x12, flag(DEVANAGARI) }, |
| /* devanagari, gujarati letter O */ |
| { 0x05, 0x3E, 0x47, 0x13, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari letter AI, gujarati letter AU */ |
| { 0x05, 0x3E, 0x48, 0x14, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari, bengali, gurmukhi, gujarati, oriya AA */ |
| { 0x05, 0x3E, -1, 0x06, flag(DEVANAGARI) | flag(BENGALI) | flag(GURMUKHI) | flag(GUJARATI) | flag(ORIYA) }, |
| /* devanagari letter candra A */ |
| { 0x05, 0x45, -1, 0x72, flag(DEVANAGARI) }, |
| /* gujarati vowel candra E */ |
| { 0x05, 0x45, -1, 0x0D, flag(GUJARATI) }, |
| /* devanagari letter short A */ |
| { 0x05, 0x46, -1, 0x04, flag(DEVANAGARI) }, |
| /* gujarati letter E */ |
| { 0x05, 0x47, -1, 0x0F, flag(GUJARATI) }, |
| /* gurmukhi, gujarati letter AI */ |
| { 0x05, 0x48, -1, 0x10, flag(GURMUKHI) | flag(GUJARATI) }, |
| /* devanagari, gujarati vowel candra O */ |
| { 0x05, 0x49, -1, 0x11, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari short O */ |
| { 0x05, 0x4A, -1, 0x12, flag(DEVANAGARI) }, |
| /* devanagari, gujarati letter O */ |
| { 0x05, 0x4B, -1, 0x13, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari letter AI, gurmukhi letter AU, gujarati letter AU */ |
| { 0x05, 0x4C, -1, 0x14, flag(DEVANAGARI) | flag(GURMUKHI) | flag(GUJARATI) }, |
| /* devanagari, gujarati vowel candra O */ |
| { 0x06, 0x45, -1, 0x11, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari short O */ |
| { 0x06, 0x46, -1, 0x12, flag(DEVANAGARI) }, |
| /* devanagari, gujarati letter O */ |
| { 0x06, 0x47, -1, 0x13, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari letter AI, gujarati letter AU */ |
| { 0x06, 0x48, -1, 0x14, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* malayalam letter II */ |
| { 0x07, 0x57, -1, 0x08, flag(MALAYALAM) }, |
| /* devanagari letter UU */ |
| { 0x09, 0x41, -1, 0x0A, flag(DEVANAGARI) }, |
| /* tamil, malayalam letter UU (some styles) */ |
| { 0x09, 0x57, -1, 0x0A, flag(TAMIL) | flag(MALAYALAM) }, |
| /* malayalam letter AI */ |
| { 0x0E, 0x46, -1, 0x10, flag(MALAYALAM) }, |
| /* devanagari candra E */ |
| { 0x0F, 0x45, -1, 0x0D, flag(DEVANAGARI) }, |
| /* devanagari short E */ |
| { 0x0F, 0x46, -1, 0x0E, flag(DEVANAGARI) }, |
| /* devanagari AI */ |
| { 0x0F, 0x47, -1, 0x10, flag(DEVANAGARI) }, |
| /* oriya AI */ |
| { 0x0F, 0x57, -1, 0x10, flag(ORIYA) }, |
| /* malayalam letter OO */ |
| { 0x12, 0x3E, -1, 0x13, flag(MALAYALAM) }, |
| /* telugu, kannada letter AU */ |
| { 0x12, 0x4C, -1, 0x14, flag(TELUGU) | flag(KANNADA) }, |
| /* telugu letter OO */ |
| { 0x12, 0x55, -1, 0x13, flag(TELUGU) }, |
| /* tamil, malayalam letter AU */ |
| { 0x12, 0x57, -1, 0x14, flag(TAMIL) | flag(MALAYALAM) }, |
| /* oriya letter AU */ |
| { 0x13, 0x57, -1, 0x14, flag(ORIYA) }, |
| /* devanagari qa */ |
| { 0x15, 0x3C, -1, 0x58, flag(DEVANAGARI) }, |
| /* devanagari, gurmukhi khha */ |
| { 0x16, 0x3C, -1, 0x59, flag(DEVANAGARI) | flag(GURMUKHI) }, |
| /* devanagari, gurmukhi ghha */ |
| { 0x17, 0x3C, -1, 0x5A, flag(DEVANAGARI) | flag(GURMUKHI) }, |
| /* devanagari, gurmukhi za */ |
| { 0x1C, 0x3C, -1, 0x5B, flag(DEVANAGARI) | flag(GURMUKHI) }, |
| /* devanagari dddha, bengali, oriya rra */ |
| { 0x21, 0x3C, -1, 0x5C, flag(DEVANAGARI) | flag(BENGALI) | flag(ORIYA) }, |
| /* devanagari, bengali, oriya rha */ |
| { 0x22, 0x3C, -1, 0x5D, flag(DEVANAGARI) | flag(BENGALI) | flag(ORIYA) }, |
| /* malayalam chillu nn */ |
| { 0x23, 0x4D, 0xFF, 0x7A, flag(MALAYALAM) }, |
| /* bengali khanda ta */ |
| { 0x24, 0x4D, 0xFF, 0x4E, flag(BENGALI) }, |
| /* devanagari nnna */ |
| { 0x28, 0x3C, -1, 0x29, flag(DEVANAGARI) }, |
| /* malayalam chillu n */ |
| { 0x28, 0x4D, 0xFF, 0x7B, flag(MALAYALAM) }, |
| /* devanagari, gurmukhi fa */ |
| { 0x2B, 0x3C, -1, 0x5E, flag(DEVANAGARI) | flag(GURMUKHI) }, |
| /* devanagari, bengali yya */ |
| { 0x2F, 0x3C, -1, 0x5F, flag(DEVANAGARI) | flag(BENGALI) }, |
| /* telugu letter vocalic R */ |
| { 0x2C, 0x41, 0x41, 0x0B, flag(TELUGU) }, |
| /* devanagari rra */ |
| { 0x30, 0x3C, -1, 0x31, flag(DEVANAGARI) }, |
| /* malayalam chillu rr */ |
| { 0x30, 0x4D, 0xFF, 0x7C, flag(MALAYALAM) }, |
| /* malayalam chillu l */ |
| { 0x32, 0x4D, 0xFF, 0x7D, flag(MALAYALAM) }, |
| /* devanagari llla */ |
| { 0x33, 0x3C, -1, 0x34, flag(DEVANAGARI) }, |
| /* malayalam chillu ll */ |
| { 0x33, 0x4D, 0xFF, 0x7E, flag(MALAYALAM) }, |
| /* telugu letter MA */ |
| { 0x35, 0x41, -1, 0x2E, flag(TELUGU) }, |
| /* devanagari, gujarati vowel sign candra O */ |
| { 0x3E, 0x45, -1, 0x49, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari vowel sign short O */ |
| { 0x3E, 0x46, -1, 0x4A, flag(DEVANAGARI) }, |
| /* devanagari, gujarati vowel sign O */ |
| { 0x3E, 0x47, -1, 0x4B, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* devanagari, gujarati vowel sign AU */ |
| { 0x3E, 0x48, -1, 0x4C, flag(DEVANAGARI) | flag(GUJARATI) }, |
| /* kannada vowel sign II */ |
| { 0x3F, 0x55, -1, 0x40, flag(KANNADA) }, |
| /* gurmukhi vowel sign UU (when stacking) */ |
| { 0x41, 0x41, -1, 0x42, flag(GURMUKHI) }, |
| /* tamil, malayalam vowel sign O */ |
| { 0x46, 0x3E, -1, 0x4A, flag(TAMIL) | flag(MALAYALAM) }, |
| /* kannada vowel sign OO */ |
| { 0x46, 0x42, 0x55, 0x4B, flag(KANNADA) }, |
| /* kannada vowel sign O */ |
| { 0x46, 0x42, -1, 0x4A, flag(KANNADA) }, |
| /* malayalam vowel sign AI (if reordered twice) */ |
| { 0x46, 0x46, -1, 0x48, flag(MALAYALAM) }, |
| /* telugu, kannada vowel sign EE */ |
| { 0x46, 0x55, -1, 0x47, flag(TELUGU) | flag(KANNADA) }, |
| /* telugu, kannada vowel sign AI */ |
| { 0x46, 0x56, -1, 0x48, flag(TELUGU) | flag(KANNADA) }, |
| /* tamil, malayalam vowel sign AU */ |
| { 0x46, 0x57, -1, 0x4C, flag(TAMIL) | flag(MALAYALAM) }, |
| /* bengali, oriya vowel sign O, tamil, malayalam vowel sign OO */ |
| { 0x47, 0x3E, -1, 0x4B, flag(BENGALI) | flag(ORIYA) | flag(TAMIL) | flag(MALAYALAM) }, |
| /* bengali, oriya vowel sign AU */ |
| { 0x47, 0x57, -1, 0x4C, flag(BENGALI) | flag(ORIYA) }, |
| /* kannada vowel sign OO */ |
| { 0x4A, 0x55, -1, 0x4B, flag(KANNADA) }, |
| /* gurmukhi letter I */ |
| { 0x72, 0x3F, -1, 0x07, flag(GURMUKHI) }, |
| /* gurmukhi letter II */ |
| { 0x72, 0x40, -1, 0x08, flag(GURMUKHI) }, |
| /* gurmukhi letter EE */ |
| { 0x72, 0x47, -1, 0x0F, flag(GURMUKHI) }, |
| /* gurmukhi letter U */ |
| { 0x73, 0x41, -1, 0x09, flag(GURMUKHI) }, |
| /* gurmukhi letter UU */ |
| { 0x73, 0x42, -1, 0x0A, flag(GURMUKHI) }, |
| /* gurmukhi letter OO */ |
| { 0x73, 0x4B, -1, 0x13, flag(GURMUKHI) }, |
| }; |
| |
| static { |
| for (ScriptData sd : scripts.values()) { |
| sd.decompMask = new BitSet(0x7F); |
| for (int i = 0; i < decompositions.length; i++) { |
| final int ch = decompositions[i][0]; |
| final int flags = decompositions[i][4]; |
| if ((flags & sd.flag) != 0) |
| sd.decompMask.set(ch); |
| } |
| } |
| } |
| |
| /** |
| * Normalizes input text, and returns the new length. |
| * The length will always be less than or equal to the existing length. |
| * |
| * @param text input text |
| * @param len valid length |
| * @return normalized length |
| */ |
| public int normalize(char text[], int len) { |
| for (int i = 0; i < len; i++) { |
| final Character.UnicodeBlock block = Character.UnicodeBlock.of(text[i]); |
| final ScriptData sd = scripts.get(block); |
| if (sd != null) { |
| final int ch = text[i] - sd.base; |
| if (sd.decompMask.get(ch)) |
| len = compose(ch, block, sd, text, i, len); |
| } |
| } |
| return len; |
| } |
| |
| /** |
| * Compose into standard form any compositions in the decompositions table. |
| */ |
| private int compose(int ch0, Character.UnicodeBlock block0, ScriptData sd, |
| char text[], int pos, int len) { |
| if (pos + 1 >= len) /* need at least 2 chars! */ |
| return len; |
| |
| final int ch1 = text[pos + 1] - sd.base; |
| final Character.UnicodeBlock block1 = Character.UnicodeBlock.of(text[pos + 1]); |
| if (block1 != block0) /* needs to be the same writing system */ |
| return len; |
| |
| int ch2 = -1; |
| |
| if (pos + 2 < len) { |
| ch2 = text[pos + 2] - sd.base; |
| Character.UnicodeBlock block2 = Character.UnicodeBlock.of(text[pos + 2]); |
| if (text[pos + 2] == '\u200D') // ZWJ |
| ch2 = 0xFF; |
| else if (block2 != block1) // still allow a 2-char match |
| ch2 = -1; |
| } |
| |
| for (int i = 0; i < decompositions.length; i++) |
| if (decompositions[i][0] == ch0 && (decompositions[i][4] & sd.flag) != 0) { |
| if (decompositions[i][1] == ch1 && (decompositions[i][2] < 0 || decompositions[i][2] == ch2)) { |
| text[pos] = (char) (sd.base + decompositions[i][3]); |
| len = delete(text, pos + 1, len); |
| if (decompositions[i][2] >= 0) |
| len = delete(text, pos + 1, len); |
| return len; |
| } |
| } |
| |
| return len; |
| } |
| } |
