lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.lucene.analysis.in;

 import static java.lang.Character.UnicodeBlock.*;
 import static org.apache.lucene.analysis.util.StemmerUtil.*;

 import java.util.BitSet;
 import java.util.IdentityHashMap;

 /**
  * Normalizes the Unicode representation of text in Indian languages.
  *
  * <p>Follows guidelines from Unicode 5.2, chapter 6, South Asian Scripts I and graphical
  * decompositions from http://ldc.upenn.edu/myl/IndianScriptsUnicode.html
  */
 public class IndicNormalizer {

   private static class ScriptData {
     final int flag;
     final int base;
     BitSet decompMask;

     ScriptData(int flag, int base) {
       this.flag = flag;
       this.base = base;
     }
   }

   private static final IdentityHashMap<Character.UnicodeBlock, ScriptData> scripts =
       new IdentityHashMap<>(9);

   private static int flag(Character.UnicodeBlock ub) {
     return scripts.get(ub).flag;
   }

   static {
     scripts.put(DEVANAGARI, new ScriptData(1, 0x0900));
     scripts.put(BENGALI, new ScriptData(2, 0x0980));
     scripts.put(GURMUKHI, new ScriptData(4, 0x0A00));
     scripts.put(GUJARATI, new ScriptData(8, 0x0A80));
     scripts.put(ORIYA, new ScriptData(16, 0x0B00));
     scripts.put(TAMIL, new ScriptData(32, 0x0B80));
     scripts.put(TELUGU, new ScriptData(64, 0x0C00));
     scripts.put(KANNADA, new ScriptData(128, 0x0C80));
     scripts.put(MALAYALAM, new ScriptData(256, 0x0D00));
   }

   /**
    * Decompositions according to Unicode 5.2, and http://ldc.upenn.edu/myl/IndianScriptsUnicode.html
    *
    * <p>Most of these are not handled by unicode normalization anyway.
    *
    * <p>The numbers here represent offsets into the respective codepages, with -1 representing null
    * and 0xFF representing zero-width joiner.
    *
    * <p>the columns are: ch1, ch2, ch3, res, flags ch1, ch2, and ch3 are the decomposition res is
    * the composition, and flags are the scripts to which it applies.
    */
   private static final int decompositions[][] = {
     /* devanagari, gujarati vowel candra O */
     {0x05, 0x3E, 0x45, 0x11, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari short O */
     {0x05, 0x3E, 0x46, 0x12, flag(DEVANAGARI)},
     /* devanagari, gujarati letter O */
     {0x05, 0x3E, 0x47, 0x13, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari letter AI, gujarati letter AU */
     {0x05, 0x3E, 0x48, 0x14, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari, bengali, gurmukhi, gujarati, oriya AA */
     {
       0x05,
       0x3E,
       -1,
       0x06,
       flag(DEVANAGARI) | flag(BENGALI) | flag(GURMUKHI) | flag(GUJARATI) | flag(ORIYA)
     },
     /* devanagari letter candra A */
     {0x05, 0x45, -1, 0x72, flag(DEVANAGARI)},
     /* gujarati vowel candra E */
     {0x05, 0x45, -1, 0x0D, flag(GUJARATI)},
     /* devanagari letter short A */
     {0x05, 0x46, -1, 0x04, flag(DEVANAGARI)},
     /* gujarati letter E */
     {0x05, 0x47, -1, 0x0F, flag(GUJARATI)},
     /* gurmukhi, gujarati letter AI */
     {0x05, 0x48, -1, 0x10, flag(GURMUKHI) | flag(GUJARATI)},
     /* devanagari, gujarati vowel candra O */
     {0x05, 0x49, -1, 0x11, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari short O */
     {0x05, 0x4A, -1, 0x12, flag(DEVANAGARI)},
     /* devanagari, gujarati letter O */
     {0x05, 0x4B, -1, 0x13, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari letter AI, gurmukhi letter AU, gujarati letter AU */
     {0x05, 0x4C, -1, 0x14, flag(DEVANAGARI) | flag(GURMUKHI) | flag(GUJARATI)},
     /* devanagari, gujarati vowel candra O */
     {0x06, 0x45, -1, 0x11, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari short O */
     {0x06, 0x46, -1, 0x12, flag(DEVANAGARI)},
     /* devanagari, gujarati letter O */
     {0x06, 0x47, -1, 0x13, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari letter AI, gujarati letter AU */
     {0x06, 0x48, -1, 0x14, flag(DEVANAGARI) | flag(GUJARATI)},
     /* malayalam letter II */
     {0x07, 0x57, -1, 0x08, flag(MALAYALAM)},
     /* devanagari letter UU */
     {0x09, 0x41, -1, 0x0A, flag(DEVANAGARI)},
     /* tamil, malayalam letter UU (some styles) */
     {0x09, 0x57, -1, 0x0A, flag(TAMIL) | flag(MALAYALAM)},
     /* malayalam letter AI */
     {0x0E, 0x46, -1, 0x10, flag(MALAYALAM)},
     /* devanagari candra E */
     {0x0F, 0x45, -1, 0x0D, flag(DEVANAGARI)},
     /* devanagari short E */
     {0x0F, 0x46, -1, 0x0E, flag(DEVANAGARI)},
     /* devanagari AI */
     {0x0F, 0x47, -1, 0x10, flag(DEVANAGARI)},
     /* oriya AI */
     {0x0F, 0x57, -1, 0x10, flag(ORIYA)},
     /* malayalam letter OO */
     {0x12, 0x3E, -1, 0x13, flag(MALAYALAM)},
     /* telugu, kannada letter AU */
     {0x12, 0x4C, -1, 0x14, flag(TELUGU) | flag(KANNADA)},
     /* telugu letter OO */
     {0x12, 0x55, -1, 0x13, flag(TELUGU)},
     /* tamil, malayalam letter AU */
     {0x12, 0x57, -1, 0x14, flag(TAMIL) | flag(MALAYALAM)},
     /* oriya letter AU */
     {0x13, 0x57, -1, 0x14, flag(ORIYA)},
     /* devanagari qa */
     {0x15, 0x3C, -1, 0x58, flag(DEVANAGARI)},
     /* devanagari, gurmukhi khha */
     {0x16, 0x3C, -1, 0x59, flag(DEVANAGARI) | flag(GURMUKHI)},
     /* devanagari, gurmukhi ghha */
     {0x17, 0x3C, -1, 0x5A, flag(DEVANAGARI) | flag(GURMUKHI)},
     /* devanagari, gurmukhi za */
     {0x1C, 0x3C, -1, 0x5B, flag(DEVANAGARI) | flag(GURMUKHI)},
     /* devanagari dddha, bengali, oriya rra */
     {0x21, 0x3C, -1, 0x5C, flag(DEVANAGARI) | flag(BENGALI) | flag(ORIYA)},
     /* devanagari, bengali, oriya rha */
     {0x22, 0x3C, -1, 0x5D, flag(DEVANAGARI) | flag(BENGALI) | flag(ORIYA)},
     /* malayalam chillu nn */
     {0x23, 0x4D, 0xFF, 0x7A, flag(MALAYALAM)},
     /* bengali khanda ta */
     {0x24, 0x4D, 0xFF, 0x4E, flag(BENGALI)},
     /* devanagari nnna */
     {0x28, 0x3C, -1, 0x29, flag(DEVANAGARI)},
     /* malayalam chillu n */
     {0x28, 0x4D, 0xFF, 0x7B, flag(MALAYALAM)},
     /* devanagari, gurmukhi fa */
     {0x2B, 0x3C, -1, 0x5E, flag(DEVANAGARI) | flag(GURMUKHI)},
     /* devanagari, bengali yya */
     {0x2F, 0x3C, -1, 0x5F, flag(DEVANAGARI) | flag(BENGALI)},
     /* telugu letter vocalic R */
     {0x2C, 0x41, 0x41, 0x0B, flag(TELUGU)},
     /* devanagari rra */
     {0x30, 0x3C, -1, 0x31, flag(DEVANAGARI)},
     /* malayalam chillu rr */
     {0x30, 0x4D, 0xFF, 0x7C, flag(MALAYALAM)},
     /* malayalam chillu l */
     {0x32, 0x4D, 0xFF, 0x7D, flag(MALAYALAM)},
     /* devanagari llla */
     {0x33, 0x3C, -1, 0x34, flag(DEVANAGARI)},
     /* malayalam chillu ll */
     {0x33, 0x4D, 0xFF, 0x7E, flag(MALAYALAM)},
     /* telugu letter MA */
     {0x35, 0x41, -1, 0x2E, flag(TELUGU)},
     /* devanagari, gujarati vowel sign candra O */
     {0x3E, 0x45, -1, 0x49, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari vowel sign short O */
     {0x3E, 0x46, -1, 0x4A, flag(DEVANAGARI)},
     /* devanagari, gujarati vowel sign O */
     {0x3E, 0x47, -1, 0x4B, flag(DEVANAGARI) | flag(GUJARATI)},
     /* devanagari, gujarati vowel sign AU */
     {0x3E, 0x48, -1, 0x4C, flag(DEVANAGARI) | flag(GUJARATI)},
     /* kannada vowel sign II */
     {0x3F, 0x55, -1, 0x40, flag(KANNADA)},
     /* gurmukhi vowel sign UU (when stacking) */
     {0x41, 0x41, -1, 0x42, flag(GURMUKHI)},
     /* tamil, malayalam vowel sign O */
     {0x46, 0x3E, -1, 0x4A, flag(TAMIL) | flag(MALAYALAM)},
     /* kannada vowel sign OO */
     {0x46, 0x42, 0x55, 0x4B, flag(KANNADA)},
     /* kannada vowel sign O */
     {0x46, 0x42, -1, 0x4A, flag(KANNADA)},
     /* malayalam vowel sign AI (if reordered twice) */
     {0x46, 0x46, -1, 0x48, flag(MALAYALAM)},
     /* telugu, kannada vowel sign EE */
     {0x46, 0x55, -1, 0x47, flag(TELUGU) | flag(KANNADA)},
     /* telugu, kannada vowel sign AI */
     {0x46, 0x56, -1, 0x48, flag(TELUGU) | flag(KANNADA)},
     /* tamil, malayalam vowel sign AU */
     {0x46, 0x57, -1, 0x4C, flag(TAMIL) | flag(MALAYALAM)},
     /* bengali, oriya vowel sign O, tamil, malayalam vowel sign OO */
     {0x47, 0x3E, -1, 0x4B, flag(BENGALI) | flag(ORIYA) | flag(TAMIL) | flag(MALAYALAM)},
     /* bengali, oriya vowel sign AU */
     {0x47, 0x57, -1, 0x4C, flag(BENGALI) | flag(ORIYA)},
     /* kannada vowel sign OO */
     {0x4A, 0x55, -1, 0x4B, flag(KANNADA)},
     /* gurmukhi letter I */
     {0x72, 0x3F, -1, 0x07, flag(GURMUKHI)},
     /* gurmukhi letter II */
     {0x72, 0x40, -1, 0x08, flag(GURMUKHI)},
     /* gurmukhi letter EE */
     {0x72, 0x47, -1, 0x0F, flag(GURMUKHI)},
     /* gurmukhi letter U */
     {0x73, 0x41, -1, 0x09, flag(GURMUKHI)},
     /* gurmukhi letter UU */
     {0x73, 0x42, -1, 0x0A, flag(GURMUKHI)},
     /* gurmukhi letter OO */
     {0x73, 0x4B, -1, 0x13, flag(GURMUKHI)},
   };

   static {
     for (ScriptData sd : scripts.values()) {
       sd.decompMask = new BitSet(0x7F);
       for (int i = 0; i < decompositions.length; i++) {
         final int ch = decompositions[i][0];
         final int flags = decompositions[i][4];
         if ((flags & sd.flag) != 0) sd.decompMask.set(ch);
       }
     }
   }

   /**
    * Normalizes input text, and returns the new length. The length will always be less than or equal
    * to the existing length.
    *
    * @param text input text
    * @param len valid length
    * @return normalized length
    */
   public int normalize(char text[], int len) {
     for (int i = 0; i < len; i++) {
       final Character.UnicodeBlock block = Character.UnicodeBlock.of(text[i]);
       final ScriptData sd = scripts.get(block);
       if (sd != null) {
         final int ch = text[i] - sd.base;
         if (sd.decompMask.get(ch)) len = compose(ch, block, sd, text, i, len);
       }
     }
     return len;
   }

   /** Compose into standard form any compositions in the decompositions table. */
   private int compose(
       int ch0, Character.UnicodeBlock block0, ScriptData sd, char text[], int pos, int len) {
     if (pos + 1 >= len) /* need at least 2 chars! */ return len;

     final int ch1 = text[pos + 1] - sd.base;
     final Character.UnicodeBlock block1 = Character.UnicodeBlock.of(text[pos + 1]);
     if (block1 != block0) /* needs to be the same writing system */ return len;

     int ch2 = -1;

     if (pos + 2 < len) {
       ch2 = text[pos + 2] - sd.base;
       Character.UnicodeBlock block2 = Character.UnicodeBlock.of(text[pos + 2]);
       if (text[pos + 2] == '\u200D') // ZWJ
       ch2 = 0xFF;
       else if (block2 != block1) // still allow a 2-char match
       ch2 = -1;
     }

     for (int i = 0; i < decompositions.length; i++)
       if (decompositions[i][0] == ch0 && (decompositions[i][4] & sd.flag) != 0) {
         if (decompositions[i][1] == ch1
             && (decompositions[i][2] < 0 || decompositions[i][2] == ch2)) {
           text[pos] = (char) (sd.base + decompositions[i][3]);
           len = delete(text, pos + 1, len);
           if (decompositions[i][2] >= 0) len = delete(text, pos + 1, len);
           return len;
         }
       }

     return len;
   }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.lucene.analysis.in;

	import static java.lang.Character.UnicodeBlock.*;
	import static org.apache.lucene.analysis.util.StemmerUtil.*;

	import java.util.BitSet;
	import java.util.IdentityHashMap;

	/**
	* Normalizes the Unicode representation of text in Indian languages.
	*
	* <p>Follows guidelines from Unicode 5.2, chapter 6, South Asian Scripts I and graphical
	* decompositions from http://ldc.upenn.edu/myl/IndianScriptsUnicode.html
	*/
	public class IndicNormalizer {

	private static class ScriptData {
	final int flag;
	final int base;
	BitSet decompMask;

	ScriptData(int flag, int base) {
	this.flag = flag;
	this.base = base;
	}
	}

	private static final IdentityHashMap<Character.UnicodeBlock, ScriptData> scripts =
	new IdentityHashMap<>(9);

	private static int flag(Character.UnicodeBlock ub) {
	return scripts.get(ub).flag;
	}

	static {
	scripts.put(DEVANAGARI, new ScriptData(1, 0x0900));
	scripts.put(BENGALI, new ScriptData(2, 0x0980));
	scripts.put(GURMUKHI, new ScriptData(4, 0x0A00));
	scripts.put(GUJARATI, new ScriptData(8, 0x0A80));
	scripts.put(ORIYA, new ScriptData(16, 0x0B00));
	scripts.put(TAMIL, new ScriptData(32, 0x0B80));
	scripts.put(TELUGU, new ScriptData(64, 0x0C00));
	scripts.put(KANNADA, new ScriptData(128, 0x0C80));
	scripts.put(MALAYALAM, new ScriptData(256, 0x0D00));
	}

	/**
	* Decompositions according to Unicode 5.2, and http://ldc.upenn.edu/myl/IndianScriptsUnicode.html
	*
	* <p>Most of these are not handled by unicode normalization anyway.
	*
	* <p>The numbers here represent offsets into the respective codepages, with -1 representing null
	* and 0xFF representing zero-width joiner.
	*
	* <p>the columns are: ch1, ch2, ch3, res, flags ch1, ch2, and ch3 are the decomposition res is
	* the composition, and flags are the scripts to which it applies.
	*/
	private static final int decompositions[][] = {
	/* devanagari, gujarati vowel candra O */
	{0x05, 0x3E, 0x45, 0x11, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari short O */
	{0x05, 0x3E, 0x46, 0x12, flag(DEVANAGARI)},
	/* devanagari, gujarati letter O */
	{0x05, 0x3E, 0x47, 0x13, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari letter AI, gujarati letter AU */
	{0x05, 0x3E, 0x48, 0x14, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari, bengali, gurmukhi, gujarati, oriya AA */
	{
	0x05,
	0x3E,
	-1,
	0x06,
	flag(DEVANAGARI) \| flag(BENGALI) \| flag(GURMUKHI) \| flag(GUJARATI) \| flag(ORIYA)
	},
	/* devanagari letter candra A */
	{0x05, 0x45, -1, 0x72, flag(DEVANAGARI)},
	/* gujarati vowel candra E */
	{0x05, 0x45, -1, 0x0D, flag(GUJARATI)},
	/* devanagari letter short A */
	{0x05, 0x46, -1, 0x04, flag(DEVANAGARI)},
	/* gujarati letter E */
	{0x05, 0x47, -1, 0x0F, flag(GUJARATI)},
	/* gurmukhi, gujarati letter AI */
	{0x05, 0x48, -1, 0x10, flag(GURMUKHI) \| flag(GUJARATI)},
	/* devanagari, gujarati vowel candra O */
	{0x05, 0x49, -1, 0x11, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari short O */
	{0x05, 0x4A, -1, 0x12, flag(DEVANAGARI)},
	/* devanagari, gujarati letter O */
	{0x05, 0x4B, -1, 0x13, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari letter AI, gurmukhi letter AU, gujarati letter AU */
	{0x05, 0x4C, -1, 0x14, flag(DEVANAGARI) \| flag(GURMUKHI) \| flag(GUJARATI)},
	/* devanagari, gujarati vowel candra O */
	{0x06, 0x45, -1, 0x11, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari short O */
	{0x06, 0x46, -1, 0x12, flag(DEVANAGARI)},
	/* devanagari, gujarati letter O */
	{0x06, 0x47, -1, 0x13, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari letter AI, gujarati letter AU */
	{0x06, 0x48, -1, 0x14, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* malayalam letter II */
	{0x07, 0x57, -1, 0x08, flag(MALAYALAM)},
	/* devanagari letter UU */
	{0x09, 0x41, -1, 0x0A, flag(DEVANAGARI)},
	/* tamil, malayalam letter UU (some styles) */
	{0x09, 0x57, -1, 0x0A, flag(TAMIL) \| flag(MALAYALAM)},
	/* malayalam letter AI */
	{0x0E, 0x46, -1, 0x10, flag(MALAYALAM)},
	/* devanagari candra E */
	{0x0F, 0x45, -1, 0x0D, flag(DEVANAGARI)},
	/* devanagari short E */
	{0x0F, 0x46, -1, 0x0E, flag(DEVANAGARI)},
	/* devanagari AI */
	{0x0F, 0x47, -1, 0x10, flag(DEVANAGARI)},
	/* oriya AI */
	{0x0F, 0x57, -1, 0x10, flag(ORIYA)},
	/* malayalam letter OO */
	{0x12, 0x3E, -1, 0x13, flag(MALAYALAM)},
	/* telugu, kannada letter AU */
	{0x12, 0x4C, -1, 0x14, flag(TELUGU) \| flag(KANNADA)},
	/* telugu letter OO */
	{0x12, 0x55, -1, 0x13, flag(TELUGU)},
	/* tamil, malayalam letter AU */
	{0x12, 0x57, -1, 0x14, flag(TAMIL) \| flag(MALAYALAM)},
	/* oriya letter AU */
	{0x13, 0x57, -1, 0x14, flag(ORIYA)},
	/* devanagari qa */
	{0x15, 0x3C, -1, 0x58, flag(DEVANAGARI)},
	/* devanagari, gurmukhi khha */
	{0x16, 0x3C, -1, 0x59, flag(DEVANAGARI) \| flag(GURMUKHI)},
	/* devanagari, gurmukhi ghha */
	{0x17, 0x3C, -1, 0x5A, flag(DEVANAGARI) \| flag(GURMUKHI)},
	/* devanagari, gurmukhi za */
	{0x1C, 0x3C, -1, 0x5B, flag(DEVANAGARI) \| flag(GURMUKHI)},
	/* devanagari dddha, bengali, oriya rra */
	{0x21, 0x3C, -1, 0x5C, flag(DEVANAGARI) \| flag(BENGALI) \| flag(ORIYA)},
	/* devanagari, bengali, oriya rha */
	{0x22, 0x3C, -1, 0x5D, flag(DEVANAGARI) \| flag(BENGALI) \| flag(ORIYA)},
	/* malayalam chillu nn */
	{0x23, 0x4D, 0xFF, 0x7A, flag(MALAYALAM)},
	/* bengali khanda ta */
	{0x24, 0x4D, 0xFF, 0x4E, flag(BENGALI)},
	/* devanagari nnna */
	{0x28, 0x3C, -1, 0x29, flag(DEVANAGARI)},
	/* malayalam chillu n */
	{0x28, 0x4D, 0xFF, 0x7B, flag(MALAYALAM)},
	/* devanagari, gurmukhi fa */
	{0x2B, 0x3C, -1, 0x5E, flag(DEVANAGARI) \| flag(GURMUKHI)},
	/* devanagari, bengali yya */
	{0x2F, 0x3C, -1, 0x5F, flag(DEVANAGARI) \| flag(BENGALI)},
	/* telugu letter vocalic R */
	{0x2C, 0x41, 0x41, 0x0B, flag(TELUGU)},
	/* devanagari rra */
	{0x30, 0x3C, -1, 0x31, flag(DEVANAGARI)},
	/* malayalam chillu rr */
	{0x30, 0x4D, 0xFF, 0x7C, flag(MALAYALAM)},
	/* malayalam chillu l */
	{0x32, 0x4D, 0xFF, 0x7D, flag(MALAYALAM)},
	/* devanagari llla */
	{0x33, 0x3C, -1, 0x34, flag(DEVANAGARI)},
	/* malayalam chillu ll */
	{0x33, 0x4D, 0xFF, 0x7E, flag(MALAYALAM)},
	/* telugu letter MA */
	{0x35, 0x41, -1, 0x2E, flag(TELUGU)},
	/* devanagari, gujarati vowel sign candra O */
	{0x3E, 0x45, -1, 0x49, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari vowel sign short O */
	{0x3E, 0x46, -1, 0x4A, flag(DEVANAGARI)},
	/* devanagari, gujarati vowel sign O */
	{0x3E, 0x47, -1, 0x4B, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* devanagari, gujarati vowel sign AU */
	{0x3E, 0x48, -1, 0x4C, flag(DEVANAGARI) \| flag(GUJARATI)},
	/* kannada vowel sign II */
	{0x3F, 0x55, -1, 0x40, flag(KANNADA)},
	/* gurmukhi vowel sign UU (when stacking) */
	{0x41, 0x41, -1, 0x42, flag(GURMUKHI)},
	/* tamil, malayalam vowel sign O */
	{0x46, 0x3E, -1, 0x4A, flag(TAMIL) \| flag(MALAYALAM)},
	/* kannada vowel sign OO */
	{0x46, 0x42, 0x55, 0x4B, flag(KANNADA)},
	/* kannada vowel sign O */
	{0x46, 0x42, -1, 0x4A, flag(KANNADA)},
	/* malayalam vowel sign AI (if reordered twice) */
	{0x46, 0x46, -1, 0x48, flag(MALAYALAM)},
	/* telugu, kannada vowel sign EE */
	{0x46, 0x55, -1, 0x47, flag(TELUGU) \| flag(KANNADA)},
	/* telugu, kannada vowel sign AI */
	{0x46, 0x56, -1, 0x48, flag(TELUGU) \| flag(KANNADA)},
	/* tamil, malayalam vowel sign AU */
	{0x46, 0x57, -1, 0x4C, flag(TAMIL) \| flag(MALAYALAM)},
	/* bengali, oriya vowel sign O, tamil, malayalam vowel sign OO */
	{0x47, 0x3E, -1, 0x4B, flag(BENGALI) \| flag(ORIYA) \| flag(TAMIL) \| flag(MALAYALAM)},
	/* bengali, oriya vowel sign AU */
	{0x47, 0x57, -1, 0x4C, flag(BENGALI) \| flag(ORIYA)},
	/* kannada vowel sign OO */
	{0x4A, 0x55, -1, 0x4B, flag(KANNADA)},
	/* gurmukhi letter I */
	{0x72, 0x3F, -1, 0x07, flag(GURMUKHI)},
	/* gurmukhi letter II */
	{0x72, 0x40, -1, 0x08, flag(GURMUKHI)},
	/* gurmukhi letter EE */
	{0x72, 0x47, -1, 0x0F, flag(GURMUKHI)},
	/* gurmukhi letter U */
	{0x73, 0x41, -1, 0x09, flag(GURMUKHI)},
	/* gurmukhi letter UU */
	{0x73, 0x42, -1, 0x0A, flag(GURMUKHI)},
	/* gurmukhi letter OO */
	{0x73, 0x4B, -1, 0x13, flag(GURMUKHI)},
	};

	static {
	for (ScriptData sd : scripts.values()) {
	sd.decompMask = new BitSet(0x7F);
	for (int i = 0; i < decompositions.length; i++) {
	final int ch = decompositions[i][0];
	final int flags = decompositions[i][4];
	if ((flags & sd.flag) != 0) sd.decompMask.set(ch);
	}
	}
	}

	/**
	* Normalizes input text, and returns the new length. The length will always be less than or equal
	* to the existing length.
	*
	* @param text input text
	* @param len valid length
	* @return normalized length
	*/
	public int normalize(char text[], int len) {
	for (int i = 0; i < len; i++) {
	final Character.UnicodeBlock block = Character.UnicodeBlock.of(text[i]);
	final ScriptData sd = scripts.get(block);
	if (sd != null) {
	final int ch = text[i] - sd.base;
	if (sd.decompMask.get(ch)) len = compose(ch, block, sd, text, i, len);
	}
	}
	return len;
	}

	/** Compose into standard form any compositions in the decompositions table. */
	private int compose(
	int ch0, Character.UnicodeBlock block0, ScriptData sd, char text[], int pos, int len) {
	if (pos + 1 >= len) /* need at least 2 chars! */ return len;

	final int ch1 = text[pos + 1] - sd.base;
	final Character.UnicodeBlock block1 = Character.UnicodeBlock.of(text[pos + 1]);
	if (block1 != block0) /* needs to be the same writing system */ return len;

	int ch2 = -1;

	if (pos + 2 < len) {
	ch2 = text[pos + 2] - sd.base;
	Character.UnicodeBlock block2 = Character.UnicodeBlock.of(text[pos + 2]);
	if (text[pos + 2] == '\u200D') // ZWJ
	ch2 = 0xFF;
	else if (block2 != block1) // still allow a 2-char match
	ch2 = -1;
	}

	for (int i = 0; i < decompositions.length; i++)
	if (decompositions[i][0] == ch0 && (decompositions[i][4] & sd.flag) != 0) {
	if (decompositions[i][1] == ch1
	&& (decompositions[i][2] < 0 \|\| decompositions[i][2] == ch2)) {
	text[pos] = (char) (sd.base + decompositions[i][3]);
	len = delete(text, pos + 1, len);
	if (decompositions[i][2] >= 0) len = delete(text, pos + 1, len);
	return len;
	}
	}

	return len;
	}
	}