| Index: lucene/src/test/org/apache/lucene/TestExternalCodecs.java |
| =================================================================== |
| --- lucene/src/test/org/apache/lucene/TestExternalCodecs.java (revision 956375) |
| +++ lucene/src/test/org/apache/lucene/TestExternalCodecs.java (working copy) |
| @@ -179,7 +179,7 @@ |
| |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + return BytesRef.getUTF8SortedAsUnicodeComparator(); |
| } |
| |
| @Override |
| @@ -263,7 +263,7 @@ |
| |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + return BytesRef.getUTF8SortedAsUnicodeComparator(); |
| } |
| |
| @Override |
| Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java |
| =================================================================== |
| --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 956375) |
| +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) |
| @@ -4621,38 +4621,22 @@ |
| private void checkTermsOrder(IndexReader r, Set<String> allTerms, boolean isTop) throws IOException { |
| TermsEnum terms = MultiFields.getFields(r).terms("f").iterator(); |
| |
| - char[] last = new char[2]; |
| - int lastLength = 0; |
| + BytesRef last = new BytesRef(); |
| |
| Set<String> seenTerms = new HashSet<String>(); |
| |
| - UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result(); |
| while(true) { |
| final BytesRef term = terms.next(); |
| if (term == null) { |
| break; |
| } |
| - UnicodeUtil.UTF8toUTF16(term.bytes, term.offset, term.length, utf16); |
| - assertTrue(utf16.length <= 2); |
| |
| - // Make sure last term comes before current one, in |
| - // UTF16 sort order |
| - int i = 0; |
| - for(i=0;i<lastLength && i<utf16.length;i++) { |
| - assertTrue("UTF16 code unit " + termDesc(new String(utf16.result, 0, utf16.length)) + " incorrectly sorted after code unit " + termDesc(new String(last, 0, lastLength)), last[i] <= utf16.result[i]); |
| - if (last[i] < utf16.result[i]) { |
| - break; |
| - } |
| - } |
| - // Terms should not have been identical |
| - assertTrue(lastLength != utf16.length || i < lastLength); |
| + assertTrue(last.compareTo(term) < 0); |
| + last.copy(term); |
| |
| - final String s = new String(utf16.result, 0, utf16.length); |
| + final String s = term.utf8ToString(); |
| assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s)); |
| seenTerms.add(s); |
| - |
| - System.arraycopy(utf16.result, 0, last, 0, utf16.length); |
| - lastLength = utf16.length; |
| } |
| |
| if (isTop) { |
| Index: lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java |
| =================================================================== |
| --- lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java (revision 956375) |
| +++ lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java (working copy) |
| @@ -1,5 +1,22 @@ |
| package org.apache.lucene.index.codecs.intblock; |
| |
| +/** |
| + * Licensed to the Apache Software Foundation (ASF) under one or more |
| + * contributor license agreements. See the NOTICE file distributed with |
| + * this work for additional information regarding copyright ownership. |
| + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| + * (the "License"); you may not use this file except in compliance with |
| + * the License. You may obtain a copy of the License at |
| + * |
| + * http://www.apache.org/licenses/LICENSE-2.0 |
| + * |
| + * Unless required by applicable law or agreed to in writing, software |
| + * distributed under the License is distributed on an "AS IS" BASIS, |
| + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| + * See the License for the specific language governing permissions and |
| + * limitations under the License. |
| + */ |
| + |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.store.*; |
| import org.apache.lucene.index.codecs.sep.*; |
| @@ -34,7 +51,7 @@ |
| out.close(); |
| |
| IntIndexInput in = new SimpleIntBlockIndexInput(dir, "test", 128); |
| - IntIndexInput.Reader r = in.reader(); |
| + in.reader(); |
| // read no ints |
| in.close(); |
| dir.close(); |
| Index: lucene/src/test/org/apache/lucene/index/codecs/preflex/TermInfosWriter.java |
| =================================================================== |
| --- lucene/src/test/org/apache/lucene/index/codecs/preflex/TermInfosWriter.java (revision 0) |
| +++ lucene/src/test/org/apache/lucene/index/codecs/preflex/TermInfosWriter.java (revision 0) |
| @@ -0,0 +1,227 @@ |
| +package org.apache.lucene.index.codecs.preflex; |
| + |
| +/** |
| + * Licensed to the Apache Software Foundation (ASF) under one or more |
| + * contributor license agreements. See the NOTICE file distributed with |
| + * this work for additional information regarding copyright ownership. |
| + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| + * (the "License"); you may not use this file except in compliance with |
| + * the License. You may obtain a copy of the License at |
| + * |
| + * http://www.apache.org/licenses/LICENSE-2.0 |
| + * |
| + * Unless required by applicable law or agreed to in writing, software |
| + * distributed under the License is distributed on an "AS IS" BASIS, |
| + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| + * See the License for the specific language governing permissions and |
| + * limitations under the License. |
| + */ |
| + |
| + |
| +import java.io.IOException; |
| +import org.apache.lucene.store.*; |
| +import org.apache.lucene.index.*; |
| +import org.apache.lucene.util.*; |
| + |
| + |
| +/** This stores a monotonically increasing set of <Term, TermInfo> pairs in a |
| + Directory. A TermInfos can be written once, in order. */ |
| + |
| +final class TermInfosWriter { |
| + /** The file format version, a negative number. */ |
| + public static final int FORMAT = -3; |
| + |
| + // Changed strings to true utf8 with length-in-bytes not |
| + // length-in-chars |
| + public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; |
| + |
| + // NOTE: always change this if you switch to a new format! |
| + public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; |
| + |
| + private FieldInfos fieldInfos; |
| + private IndexOutput output; |
| + private TermInfo lastTi = new TermInfo(); |
| + private long size; |
| + |
| + // TODO: the default values for these two parameters should be settable from |
| + // IndexWriter. However, once that's done, folks will start setting them to |
| + // ridiculous values and complaining that things don't work well, as with |
| + // mergeFactor. So, let's wait until a number of folks find that alternate |
| + // values work better. Note that both of these values are stored in the |
| + // segment, so that it's safe to change these w/o rebuilding all indexes. |
| + |
| + /** Expert: The fraction of terms in the "dictionary" which should be stored |
| + * in RAM. Smaller values use more memory, but make searching slightly |
| + * faster, while larger values use less memory and make searching slightly |
| + * slower. Searching is typically not dominated by dictionary lookup, so |
| + * tweaking this is rarely useful.*/ |
| + int indexInterval = 128; |
| + |
| + /** Expert: The fraction of {@link TermDocs} entries stored in skip tables, |
| + * used to accelerate {@link TermDocs#skipTo(int)}. Larger values result in |
| + * smaller indexes, greater acceleration, but fewer accelerable cases, while |
| + * smaller values result in bigger indexes, less acceleration and more |
| + * accelerable cases. More detailed experiments would be useful here. */ |
| + int skipInterval = 16; |
| + |
| + /** Expert: The maximum number of skip levels. Smaller values result in |
| + * slightly smaller indexes, but slower skipping in big posting lists. |
| + */ |
| + int maxSkipLevels = 10; |
| + |
| + private long lastIndexPointer; |
| + private boolean isIndex; |
| + private byte[] lastTermBytes = new byte[10]; |
| + private int lastTermBytesLength = 0; |
| + private int lastFieldNumber = -1; |
| + |
| + private TermInfosWriter other; |
| + private BytesRef utf8Result = new BytesRef(10); |
| + |
| + TermInfosWriter(Directory directory, String segment, FieldInfos fis, |
| + int interval) |
| + throws IOException { |
| + initialize(directory, segment, fis, interval, false); |
| + other = new TermInfosWriter(directory, segment, fis, interval, true); |
| + other.other = this; |
| + } |
| + |
| + private TermInfosWriter(Directory directory, String segment, FieldInfos fis, |
| + int interval, boolean isIndex) throws IOException { |
| + initialize(directory, segment, fis, interval, isIndex); |
| + } |
| + |
| + private void initialize(Directory directory, String segment, FieldInfos fis, |
| + int interval, boolean isi) throws IOException { |
| + indexInterval = interval; |
| + fieldInfos = fis; |
| + isIndex = isi; |
| + output = directory.createOutput(segment + (isIndex ? ".tii" : ".tis")); |
| + output.writeInt(FORMAT_CURRENT); // write format |
| + output.writeLong(0); // leave space for size |
| + output.writeInt(indexInterval); // write indexInterval |
| + output.writeInt(skipInterval); // write skipInterval |
| + output.writeInt(maxSkipLevels); // write maxSkipLevels |
| + assert initUTF16Results(); |
| + } |
| + |
| + void add(Term term, TermInfo ti) throws IOException { |
| + UnicodeUtil.UTF16toUTF8(term.text(), 0, term.text().length(), utf8Result); |
| + add(fieldInfos.fieldNumber(term.field()), utf8Result.bytes, utf8Result.length, ti); |
| + } |
| + |
| + // Currently used only by assert statements |
| + UnicodeUtil.UTF16Result utf16Result1; |
| + UnicodeUtil.UTF16Result utf16Result2; |
| + |
| + // Currently used only by assert statements |
| + private boolean initUTF16Results() { |
| + utf16Result1 = new UnicodeUtil.UTF16Result(); |
| + utf16Result2 = new UnicodeUtil.UTF16Result(); |
| + return true; |
| + } |
| + |
| + // Currently used only by assert statement |
| + private int compareToLastTerm(int fieldNumber, byte[] termBytes, int termBytesLength) { |
| + |
| + if (lastFieldNumber != fieldNumber) { |
| + final int cmp = fieldInfos.fieldName(lastFieldNumber).compareTo(fieldInfos.fieldName(fieldNumber)); |
| + // If there is a field named "" (empty string) then we |
| + // will get 0 on this comparison, yet, it's "OK". But |
| + // it's not OK if two different field numbers map to |
| + // the same name. |
| + if (cmp != 0 || lastFieldNumber != -1) |
| + return cmp; |
| + } |
| + |
| + UnicodeUtil.UTF8toUTF16(lastTermBytes, 0, lastTermBytesLength, utf16Result1); |
| + UnicodeUtil.UTF8toUTF16(termBytes, 0, termBytesLength, utf16Result2); |
| + final int len; |
| + if (utf16Result1.length < utf16Result2.length) |
| + len = utf16Result1.length; |
| + else |
| + len = utf16Result2.length; |
| + |
| + for(int i=0;i<len;i++) { |
| + final char ch1 = utf16Result1.result[i]; |
| + final char ch2 = utf16Result2.result[i]; |
| + if (ch1 != ch2) |
| + return ch1-ch2; |
| + } |
| + return utf16Result1.length - utf16Result2.length; |
| + } |
| + |
| + /** Adds a new <<fieldNumber, termBytes>, TermInfo> pair to the set. |
| + Term must be lexicographically greater than all previous Terms added. |
| + TermInfo pointers must be positive and greater than all previous.*/ |
| + void add(int fieldNumber, byte[] termBytes, int termBytesLength, TermInfo ti) |
| + throws IOException { |
| + |
| + assert compareToLastTerm(fieldNumber, termBytes, termBytesLength) < 0 || |
| + (isIndex && termBytesLength == 0 && lastTermBytesLength == 0) : |
| + "Terms are out of order: field=" + fieldInfos.fieldName(fieldNumber) + " (number " + fieldNumber + ")" + |
| + " lastField=" + fieldInfos.fieldName(lastFieldNumber) + " (number " + lastFieldNumber + ")" + |
| + " text=" + new String(termBytes, 0, termBytesLength, "UTF-8") + " lastText=" + new String(lastTermBytes, 0, lastTermBytesLength, "UTF-8"); |
| + |
| + assert ti.freqPointer >= lastTi.freqPointer: "freqPointer out of order (" + ti.freqPointer + " < " + lastTi.freqPointer + ")"; |
| + assert ti.proxPointer >= lastTi.proxPointer: "proxPointer out of order (" + ti.proxPointer + " < " + lastTi.proxPointer + ")"; |
| + |
| + if (!isIndex && size % indexInterval == 0) |
| + other.add(lastFieldNumber, lastTermBytes, lastTermBytesLength, lastTi); // add an index term |
| + |
| + writeTerm(fieldNumber, termBytes, termBytesLength); // write term |
| + |
| + output.writeVInt(ti.docFreq); // write doc freq |
| + output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers |
| + output.writeVLong(ti.proxPointer - lastTi.proxPointer); |
| + |
| + if (ti.docFreq >= skipInterval) { |
| + output.writeVInt(ti.skipOffset); |
| + } |
| + |
| + if (isIndex) { |
| + output.writeVLong(other.output.getFilePointer() - lastIndexPointer); |
| + lastIndexPointer = other.output.getFilePointer(); // write pointer |
| + } |
| + |
| + lastFieldNumber = fieldNumber; |
| + lastTi.set(ti); |
| + size++; |
| + } |
| + |
| + private void writeTerm(int fieldNumber, byte[] termBytes, int termBytesLength) |
| + throws IOException { |
| + |
| + // TODO: UTF16toUTF8 could tell us this prefix |
| + // Compute prefix in common with last term: |
| + int start = 0; |
| + final int limit = termBytesLength < lastTermBytesLength ? termBytesLength : lastTermBytesLength; |
| + while(start < limit) { |
| + if (termBytes[start] != lastTermBytes[start]) |
| + break; |
| + start++; |
| + } |
| + |
| + final int length = termBytesLength - start; |
| + output.writeVInt(start); // write shared prefix length |
| + output.writeVInt(length); // write delta length |
| + output.writeBytes(termBytes, start, length); // write delta bytes |
| + output.writeVInt(fieldNumber); // write field num |
| + if (lastTermBytes.length < termBytesLength) { |
| + lastTermBytes = ArrayUtil.grow(lastTermBytes, termBytesLength); |
| + } |
| + System.arraycopy(termBytes, start, lastTermBytes, start, length); |
| + lastTermBytesLength = termBytesLength; |
| + } |
| + |
| + /** Called to complete TermInfos creation. */ |
| + void close() throws IOException { |
| + output.seek(4); // write size after format |
| + output.writeLong(size); |
| + output.close(); |
| + |
| + if (!isIndex) |
| + other.close(); |
| + } |
| + |
| +} |
| |
| Property changes on: lucene/src/test/org/apache/lucene/index/codecs/preflex/TermInfosWriter.java |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| + native |
| |
| Index: lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java |
| =================================================================== |
| --- lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java (revision 0) |
| +++ lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java (revision 0) |
| @@ -0,0 +1,206 @@ |
| +package org.apache.lucene.index.codecs.preflex; |
| + |
| +/** |
| + * Licensed to the Apache Software Foundation (ASF) under one or more |
| + * contributor license agreements. See the NOTICE file distributed with |
| + * this work for additional information regarding copyright ownership. |
| + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| + * (the "License"); you may not use this file except in compliance with |
| + * the License. You may obtain a copy of the License at |
| + * |
| + * http://www.apache.org/licenses/LICENSE-2.0 |
| + * |
| + * Unless required by applicable law or agreed to in writing, software |
| + * distributed under the License is distributed on an "AS IS" BASIS, |
| + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| + * See the License for the specific language governing permissions and |
| + * limitations under the License. |
| + */ |
| + |
| +import org.apache.lucene.store.*; |
| +import org.apache.lucene.index.*; |
| +import org.apache.lucene.index.codecs.*; |
| +import org.apache.lucene.util.*; |
| + |
| +import java.util.*; |
| +import java.io.IOException; |
| + |
| +public class TestSurrogates extends LuceneTestCase { |
| + |
| + private static final boolean DEBUG = false; |
| + |
| + // like Term, but uses BytesRef for text |
| + private static class FieldAndText implements Comparable<FieldAndText> { |
| + String field; |
| + BytesRef text; |
| + |
| + public FieldAndText(Term t) { |
| + field = t.field(); |
| + text = new BytesRef(t.text()); |
| + } |
| + |
| + public int compareTo(FieldAndText other) { |
| + if (other.field == field) { |
| + return text.compareTo(other.text); |
| + } else { |
| + return field.compareTo(other.field); |
| + } |
| + } |
| + } |
| + |
| + // chooses from a very limited alphabet to exacerbate the |
| + // surrogate seeking required |
| + private static String makeDifficultRandomUnicodeString(Random r) { |
| + final int end = r.nextInt(20); |
| + if (end == 0) { |
| + // allow 0 length |
| + return ""; |
| + } |
| + final char[] buffer = new char[end]; |
| + for (int i = 0; i < end; i++) { |
| + int t = r.nextInt(5); |
| + |
| + if (0 == t && i < end - 1) { |
| + // hi |
| + buffer[i++] = (char) 0xd800; |
| + // lo |
| + buffer[i] = (char) 0xdc00; |
| + } else if (t <= 3) { |
| + buffer[i] = 'a'; |
| + } else if (4 == t) { |
| + buffer[i] = 0xe000; |
| + } |
| + } |
| + |
| + return new String(buffer, 0, end); |
| + } |
| + |
| + private SegmentInfo makePreFlexSegment(Random r, String segName, Directory dir, FieldInfos fieldInfos, Codec codec, List<FieldAndText> fieldTerms) throws IOException { |
| + |
| + final int numField = _TestUtil.nextInt(r, 2, 5); |
| + |
| + List<Term> terms = new ArrayList<Term>(); |
| + |
| + int tc = 0; |
| + |
| + for(int f=0;f<numField;f++) { |
| + String field = "f" + f; |
| + Term protoTerm = new Term(field); |
| + |
| + fieldInfos.add(field, true, false, false, false, false, false, false); |
| + final int numTerms = 1000*_TestUtil.getRandomMultiplier(); |
| + for(int i=0;i<numTerms;i++) { |
| + String s; |
| + if (r.nextInt(3) == 1) { |
| + s = makeDifficultRandomUnicodeString(r); |
| + } else { |
| + s = _TestUtil.randomUnicodeString(r); |
| + } |
| + terms.add(protoTerm.createTerm(s + "_" + (tc++))); |
| + } |
| + } |
| + |
| + fieldInfos.write(dir, segName); |
| + |
| + // sorts in UTF16 order, just like preflex: |
| + Collections.sort(terms); |
| + |
| + TermInfosWriter w = new TermInfosWriter(dir, segName, fieldInfos, 128); |
| + TermInfo ti = new TermInfo(); |
| + BytesRef utf8 = new BytesRef(10); |
| + String lastText = null; |
| + int uniqueTermCount = 0; |
| + if (DEBUG) { |
| + System.out.println("TEST: utf16 order:"); |
| + } |
| + for(Term t : terms) { |
| + FieldInfo fi = fieldInfos.fieldInfo(t.field()); |
| + |
| + String text = t.text(); |
| + if (lastText != null && lastText.equals(text)) { |
| + continue; |
| + } |
| + fieldTerms.add(new FieldAndText(t)); |
| + uniqueTermCount++; |
| + lastText = text; |
| + UnicodeUtil.UTF16toUTF8(text, 0, text.length(), utf8); |
| + |
| + if (DEBUG) { |
| + System.out.println(" " + toHexString(t)); |
| + } |
| + w.add(fi.number, utf8.bytes, utf8.length, ti); |
| + } |
| + w.close(); |
| + |
| + Collections.sort(fieldTerms); |
| + if (DEBUG) { |
| + System.out.println("\nTEST: codepoint order"); |
| + for(FieldAndText t: fieldTerms) { |
| + System.out.println(" " + t.field + ":" + UnicodeUtil.toHexString(t.text.utf8ToString())); |
| + } |
| + } |
| + |
| + dir.createOutput(segName + ".prx").close(); |
| + dir.createOutput(segName + ".frq").close(); |
| + |
| + // !!hack alert!! stuffing uniqueTermCount in as docCount |
| + return new SegmentInfo(segName, uniqueTermCount, dir, false, -1, null, false, true, codec); |
| + } |
| + |
| + private String toHexString(Term t) { |
| + return t.field() + ":" + UnicodeUtil.toHexString(t.text()); |
| + } |
| + |
| + public void testSurrogatesOrder() throws Exception { |
| + Directory dir = new MockRAMDirectory(); |
| + |
| + Codec codec = new PreFlexCodec(); |
| + |
| + Random r = newRandom(); |
| + FieldInfos fieldInfos = new FieldInfos(); |
| + List<FieldAndText> fieldTerms = new ArrayList<FieldAndText>(); |
| + SegmentInfo si = makePreFlexSegment(r, "_0", dir, fieldInfos, codec, fieldTerms); |
| + |
| + // hack alert!! |
| + int uniqueTermCount = si.docCount; |
| + |
| + FieldsProducer fields = codec.fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, 1)); |
| + assertNotNull(fields); |
| + |
| + if (DEBUG) { |
| + System.out.println("\nTEST: now enum"); |
| + } |
| + FieldsEnum fieldsEnum = fields.iterator(); |
| + String field; |
| + UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result(); |
| + |
| + int termCount = 0; |
| + while((field = fieldsEnum.next()) != null) { |
| + TermsEnum termsEnum = fieldsEnum.terms(); |
| + BytesRef text; |
| + BytesRef lastText = null; |
| + while((text = termsEnum.next()) != null) { |
| + UnicodeUtil.UTF8toUTF16(text.bytes, text.offset, text.length, utf16); |
| + if (DEBUG) { |
| + System.out.println("got term=" + field + ":" + UnicodeUtil.toHexString(new String(utf16.result, 0, utf16.length))); |
| + System.out.println(); |
| + } |
| + if (lastText == null) { |
| + lastText = new BytesRef(text); |
| + } else { |
| + assertTrue(lastText.compareTo(text) < 0); |
| + lastText.copy(text); |
| + } |
| + assertEquals(fieldTerms.get(termCount).field, field); |
| + assertEquals(fieldTerms.get(termCount).text, text); |
| + termCount++; |
| + } |
| + if (DEBUG) { |
| + System.out.println(" no more terms for field=" + field); |
| + } |
| + } |
| + assertEquals(uniqueTermCount, termCount); |
| + |
| + fields.close(); |
| + } |
| +} |
| |
| Property changes on: lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| + native |
| |
| Index: lucene/src/test/org/apache/lucene/util/_TestUtil.java |
| =================================================================== |
| --- lucene/src/test/org/apache/lucene/util/_TestUtil.java (revision 956375) |
| +++ lucene/src/test/org/apache/lucene/util/_TestUtil.java (working copy) |
| @@ -141,7 +141,7 @@ |
| else if (t <= 1) buffer[i] = (char) r.nextInt(0x80); |
| else if (2 == t) buffer[i] = (char) nextInt(r, 0x80, 0x800); |
| else if (3 == t) buffer[i] = (char) nextInt(r, 0x800, 0xd7ff); |
| - else if (4 == t) buffer[i] = (char) nextInt(r, 0xe000, 0xffff); |
| + else if (4 == t) buffer[i] = (char) nextInt(r, 0xe000, 0xfffe); |
| } |
| return new String(buffer, 0, end); |
| } |
| Index: lucene/src/test/org/apache/lucene/util/TestNumericUtils.java |
| =================================================================== |
| --- lucene/src/test/org/apache/lucene/util/TestNumericUtils.java (revision 956375) |
| +++ lucene/src/test/org/apache/lucene/util/TestNumericUtils.java (working copy) |
| @@ -30,7 +30,7 @@ |
| NumericUtils.longToPrefixCoded(l, 0, act); |
| if (last!=null) { |
| // test if smaller |
| - assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUTF16Comparator().compare(last, act) < 0 ); |
| + assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUnicodeComparator().compare(last, act) < 0 ); |
| assertTrue("actual bigger than last (as String)", last.utf8ToString().compareTo(act.utf8ToString()) < 0 ); |
| } |
| // test is back and forward conversion works |
| @@ -48,7 +48,7 @@ |
| NumericUtils.intToPrefixCoded(i, 0, act); |
| if (last!=null) { |
| // test if smaller |
| - assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUTF16Comparator().compare(last, act) < 0 ); |
| + assertTrue("actual bigger than last (BytesRef)", BytesRef.getUTF8SortedAsUnicodeComparator().compare(last, act) < 0 ); |
| assertTrue("actual bigger than last (as String)", last.utf8ToString().compareTo(act.utf8ToString()) < 0 ); |
| } |
| // test is back and forward conversion works |
| @@ -84,7 +84,7 @@ |
| |
| // check sort order (prefixVals should be ascending) |
| for (int i=1; i<prefixVals.length; i++) { |
| - assertTrue( "check sort order", BytesRef.getUTF8SortedAsUTF16Comparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 ); |
| + assertTrue( "check sort order", BytesRef.getUTF8SortedAsUnicodeComparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 ); |
| } |
| |
| // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits |
| @@ -124,7 +124,7 @@ |
| |
| // check sort order (prefixVals should be ascending) |
| for (int i=1; i<prefixVals.length; i++) { |
| - assertTrue( "check sort order", BytesRef.getUTF8SortedAsUTF16Comparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 ); |
| + assertTrue( "check sort order", BytesRef.getUTF8SortedAsUnicodeComparator().compare(prefixVals[i-1], prefixVals[i] ) < 0 ); |
| } |
| |
| // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits |
| Index: lucene/src/java/org/apache/lucene/search/AutomatonTermsEnum.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/search/AutomatonTermsEnum.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/search/AutomatonTermsEnum.java (working copy) |
| @@ -103,7 +103,7 @@ |
| // build a cache of sorted transitions for every state |
| allTransitions = new Transition[runAutomaton.getSize()][]; |
| for (State state : this.automaton.getNumberedStates()) { |
| - state.sortTransitions(Transition.CompareByMinMaxThenDestUTF8InUTF16Order); |
| + state.sortTransitions(Transition.CompareByMinMaxThenDest); |
| state.trimTransitionsArray(); |
| allTransitions[state.getNumber()] = state.transitionsArray; |
| } |
| @@ -158,11 +158,7 @@ |
| // seek to the next possible string; |
| if (nextString()) { |
| // reposition |
| - |
| - // FIXME: this is really bad to turn off |
| - // but it cannot work correctly until terms are in utf8 order. |
| - linear = false; |
| - |
| + |
| if (linear) |
| setLinear(infinitePosition); |
| return seekBytesRef; |
| @@ -188,15 +184,15 @@ |
| } |
| for (int i = 0; i < allTransitions[state].length; i++) { |
| Transition t = allTransitions[state][i]; |
| - if (compareToUTF16(t.getMin(), (seekBytesRef.bytes[position] & 0xff)) <= 0 && |
| - compareToUTF16((seekBytesRef.bytes[position] & 0xff), t.getMax()) <= 0) { |
| + if (t.getMin() <= (seekBytesRef.bytes[position] & 0xff) && |
| + (seekBytesRef.bytes[position] & 0xff) <= t.getMax()) { |
| maxInterval = t.getMax(); |
| break; |
| } |
| } |
| - // 0xef terms don't get the optimization... not worth the trouble. |
| - if (maxInterval != 0xef) |
| - maxInterval = incrementUTF16(maxInterval); |
| + // 0xff terms don't get the optimization... not worth the trouble. |
| + if (maxInterval != 0xff) |
| + maxInterval = incrementUTF8(maxInterval); |
| int length = position + 1; /* position + maxTransition */ |
| if (linearUpperBound.bytes.length < length) |
| linearUpperBound.bytes = new byte[length]; |
| @@ -281,7 +277,7 @@ |
| // if the next character is U+FFFF and is not part of the useful portion, |
| // then by definition it puts us in a reject state, and therefore this |
| // path is dead. there cannot be any higher transitions. backtrack. |
| - c = incrementUTF16(c); |
| + c = incrementUTF8(c); |
| if (c == -1) |
| return false; |
| } |
| @@ -295,8 +291,8 @@ |
| |
| for (int i = 0; i < transitions.length; i++) { |
| Transition transition = transitions[i]; |
| - if (compareToUTF16(transition.getMax(), c) >= 0) { |
| - int nextChar = compareToUTF16(c, transition.getMin()) > 0 ? c : transition.getMin(); |
| + if (transition.getMax() >= c) { |
| + int nextChar = Math.max(c, transition.getMin()); |
| // append either the next sequential char, or the minimum transition |
| seekBytesRef.grow(seekBytesRef.length + 1); |
| seekBytesRef.length++; |
| @@ -342,9 +338,9 @@ |
| private boolean backtrack(int position) { |
| while (position > 0) { |
| int nextChar = seekBytesRef.bytes[position - 1] & 0xff; |
| - // if a character is 0xef its a dead-end too, |
| - // because there is no higher character in UTF-16 sort order. |
| - nextChar = incrementUTF16(nextChar); |
| + // if a character is 0xff its a dead-end too, |
| + // because there is no higher character in UTF-8 sort order. |
| + nextChar = incrementUTF8(nextChar); |
| if (nextChar != -1) { |
| seekBytesRef.bytes[position - 1] = (byte) nextChar; |
| seekBytesRef.length = position; |
| @@ -355,34 +351,11 @@ |
| return false; /* all solutions exhausted */ |
| } |
| |
| - /* return the next utf8 byte in utf16 order, or -1 if exhausted */ |
| - private final int incrementUTF16(int utf8) { |
| + /* return the next utf8 byte in utf8 order, or -1 if exhausted */ |
| + private final int incrementUTF8(int utf8) { |
| switch(utf8) { |
| - case 0xed: return 0xf0; |
| - case 0xfd: return 0xee; |
| - case 0xee: return 0xef; |
| - case 0xef: return -1; |
| + case 0xff: return -1; |
| default: return utf8 + 1; |
| } |
| } |
| - |
| - int compareToUTF16(int aByte, int bByte) { |
| - if (aByte != bByte) { |
| - // See http://icu-project.org/docs/papers/utf16_code_point_order.html#utf-8-in-utf-16-order |
| - |
| - // We know the terms are not equal, but, we may |
| - // have to carefully fixup the bytes at the |
| - // difference to match UTF16's sort order: |
| - if (aByte >= 0xee && bByte >= 0xee) { |
| - if ((aByte & 0xfe) == 0xee) { |
| - aByte += 0x10; |
| - } |
| - if ((bByte&0xfe) == 0xee) { |
| - bByte += 0x10; |
| - } |
| - } |
| - return aByte - bByte; |
| - } |
| - return 0; |
| - } |
| } |
| Index: lucene/src/java/org/apache/lucene/index/FieldInfos.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/FieldInfos.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/FieldInfos.java (working copy) |
| @@ -53,7 +53,7 @@ |
| private final HashMap<String,FieldInfo> byName = new HashMap<String,FieldInfo>(); |
| private int format; |
| |
| - FieldInfos() { } |
| + public FieldInfos() { } |
| |
| /** |
| * Construct a FieldInfos object using the directory and the name of the file |
| @@ -62,7 +62,7 @@ |
| * @param name The name of the file to open the IndexInput from in the Directory |
| * @throws IOException |
| */ |
| - FieldInfos(Directory d, String name) throws IOException { |
| + public FieldInfos(Directory d, String name) throws IOException { |
| IndexInput input = d.openInput(name); |
| try { |
| read(input, name); |
| Index: lucene/src/java/org/apache/lucene/index/TermsEnum.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/TermsEnum.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/TermsEnum.java (working copy) |
| @@ -144,8 +144,7 @@ |
| |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - // return an unused dummy to prevent NPE |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + return null; |
| } |
| |
| @Override |
| Index: lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (working copy) |
| @@ -130,7 +130,7 @@ |
| |
| // TODO: we may want to make this sort in same order |
| // as Codec's terms dict? |
| - final int[] termIDs = termsHashPerField.sortPostings(BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + final int[] termIDs = termsHashPerField.sortPostings(BytesRef.getUTF8SortedAsUnicodeComparator()); |
| |
| tvf.writeVInt(numPostings); |
| byte bits = 0x0; |
| Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) |
| @@ -3964,7 +3964,7 @@ |
| // commit merged deletes |
| SegmentReader reader = merge.readers[i] = readerPool.get(info, merge.mergeDocStores, |
| MERGE_READ_BUFFER_SIZE, |
| - -1); |
| + -config.getReaderTermsIndexDivisor()); |
| |
| // We clone the segment readers because other |
| // deletes may come in while we're merging so we |
| Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/CheckIndex.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/CheckIndex.java (working copy) |
| @@ -32,7 +32,7 @@ |
| import java.io.IOException; |
| import java.io.File; |
| import java.util.Collection; |
| - |
| +import java.util.Comparator; |
| import java.util.List; |
| import java.util.ArrayList; |
| import java.util.Map; |
| @@ -596,6 +596,10 @@ |
| boolean hasOrd = true; |
| final long termCountStart = status.termCount; |
| |
| + BytesRef lastTerm = null; |
| + |
| + Comparator<BytesRef> termComp = terms.getComparator(); |
| + |
| while(true) { |
| |
| final BytesRef term = terms.next(); |
| @@ -603,6 +607,17 @@ |
| break; |
| } |
| |
| + // make sure terms arrive in order according to |
| + // the comp |
| + if (lastTerm == null) { |
| + lastTerm = new BytesRef(term); |
| + } else { |
| + if (termComp.compare(lastTerm, term) >= 0) { |
| + throw new RuntimeException("terms out of order: lastTerm=" + lastTerm + " term=" + term); |
| + } |
| + lastTerm.copy(term); |
| + } |
| + |
| final int docFreq = terms.docFreq(); |
| status.totFreq += docFreq; |
| |
| Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (working copy) |
| @@ -80,7 +80,7 @@ |
| // Terms dict |
| success = false; |
| try { |
| - FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, pulsingWriter, BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, pulsingWriter, BytesRef.getUTF8SortedAsUnicodeComparator()); |
| success = true; |
| return ret; |
| } finally { |
| @@ -111,7 +111,7 @@ |
| state.fieldInfos, |
| state.segmentInfo.name, |
| state.termsIndexDivisor, |
| - BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + BytesRef.getUTF8SortedAsUnicodeComparator()); |
| success = true; |
| } finally { |
| if (!success) { |
| @@ -126,7 +126,7 @@ |
| state.dir, state.fieldInfos, state.segmentInfo.name, |
| pulsingReader, |
| state.readBufferSize, |
| - BytesRef.getUTF8SortedAsUTF16Comparator(), |
| + BytesRef.getUTF8SortedAsUnicodeComparator(), |
| StandardCodec.TERMS_CACHE_SIZE); |
| success = true; |
| return ret; |
| Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java (working copy) |
| @@ -63,7 +63,7 @@ |
| |
| success = false; |
| try { |
| - FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUnicodeComparator()); |
| success = true; |
| return ret; |
| } finally { |
| @@ -95,7 +95,7 @@ |
| state.fieldInfos, |
| state.segmentInfo.name, |
| state.termsIndexDivisor, |
| - BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + BytesRef.getUTF8SortedAsUnicodeComparator()); |
| success = true; |
| } finally { |
| if (!success) { |
| @@ -111,7 +111,7 @@ |
| state.segmentInfo.name, |
| postingsReader, |
| state.readBufferSize, |
| - BytesRef.getUTF8SortedAsUTF16Comparator(), |
| + BytesRef.getUTF8SortedAsUnicodeComparator(), |
| StandardCodec.TERMS_CACHE_SIZE); |
| success = true; |
| return ret; |
| Index: lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java (working copy) |
| @@ -104,7 +104,7 @@ |
| indexInterval = in.readInt(); |
| this.indexDivisor = indexDivisor; |
| |
| - if (indexDivisor == -1) { |
| + if (indexDivisor < 0) { |
| totalIndexInterval = indexInterval; |
| } else { |
| // In case terms index gets loaded, later, on demand |
| @@ -131,7 +131,7 @@ |
| } |
| success = true; |
| } finally { |
| - if (indexDivisor != -1) { |
| + if (indexDivisor > 0) { |
| in.close(); |
| this.in = null; |
| if (success) { |
| Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (working copy) |
| @@ -58,7 +58,7 @@ |
| |
| success = false; |
| try { |
| - FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, docs, BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, docs, BytesRef.getUTF8SortedAsUnicodeComparator()); |
| success = true; |
| return ret; |
| } finally { |
| @@ -85,7 +85,7 @@ |
| state.fieldInfos, |
| state.segmentInfo.name, |
| state.termsIndexDivisor, |
| - BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + BytesRef.getUTF8SortedAsUnicodeComparator()); |
| success = true; |
| } finally { |
| if (!success) { |
| @@ -101,7 +101,7 @@ |
| state.segmentInfo.name, |
| postings, |
| state.readBufferSize, |
| - BytesRef.getUTF8SortedAsUTF16Comparator(), |
| + BytesRef.getUTF8SortedAsUnicodeComparator(), |
| TERMS_CACHE_SIZE); |
| success = true; |
| return ret; |
| Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java (working copy) |
| @@ -53,6 +53,7 @@ |
| long indexPointer = 0; |
| int indexInterval; |
| int skipInterval; |
| + int newSuffixStart; |
| int maxSkipLevels; |
| private int formatM1SkipInterval; |
| |
| @@ -136,6 +137,7 @@ |
| |
| prevBuffer.set(termBuffer); |
| termBuffer.read(input, fieldInfos); |
| + newSuffixStart = termBuffer.newSuffixStart; |
| |
| termInfo.docFreq = input.readVInt(); // read doc freq |
| termInfo.freqPointer += input.readVLong(); // read freq pointer |
| Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java (working copy) |
| @@ -19,7 +19,6 @@ |
| |
| import java.io.IOException; |
| import org.apache.lucene.store.IndexInput; |
| -import org.apache.lucene.util.ArrayUtil; |
| import org.apache.lucene.util.BytesRef; |
| import org.apache.lucene.util.UnicodeUtil; |
| import org.apache.lucene.index.Term; |
| @@ -34,6 +33,8 @@ |
| private UnicodeUtil.UTF16Result text = new UnicodeUtil.UTF16Result(); |
| private BytesRef bytes = new BytesRef(10); |
| |
| + int newSuffixStart; |
| + |
| public final int compareTo(TermBuffer other) { |
| if (field == other.field) // fields are interned |
| return compareChars(text.result, text.length, other.text.result, other.text.length); |
| @@ -60,23 +61,33 @@ |
| int start = input.readVInt(); |
| int length = input.readVInt(); |
| int totalLength = start + length; |
| + if (bytes.bytes.length < totalLength) { |
| + bytes.grow(totalLength); |
| + } |
| if (dirty) { |
| // Fully convert all bytes since bytes is dirty |
| UnicodeUtil.UTF16toUTF8(text.result, 0, text.length, bytes); |
| - if (bytes.bytes.length < totalLength) |
| - bytes.bytes = new byte[totalLength]; |
| bytes.length = totalLength; |
| input.readBytes(bytes.bytes, start, length); |
| UnicodeUtil.UTF8toUTF16(bytes.bytes, 0, totalLength, text); |
| dirty = false; |
| } else { |
| // Incrementally convert only the UTF8 bytes that are new: |
| - if (bytes.bytes.length < totalLength) |
| - bytes.bytes = ArrayUtil.grow(bytes.bytes, totalLength); |
| bytes.length = totalLength; |
| input.readBytes(bytes.bytes, start, length); |
| UnicodeUtil.UTF8toUTF16(bytes.bytes, start, length, text); |
| } |
| + |
| + while(true) { |
| + newSuffixStart = text.offsets[start]; |
| + if (newSuffixStart != -1) { |
| + break; |
| + } |
| + if (--start == 0) { |
| + newSuffixStart = 0; |
| + break; |
| + } |
| + } |
| this.field = fieldInfos.fieldName(input.readVInt()); |
| } |
| |
| @@ -124,10 +135,11 @@ |
| try { |
| clone = (TermBuffer)super.clone(); |
| } catch (CloneNotSupportedException e) {} |
| - |
| clone.dirty = true; |
| clone.bytes = new BytesRef(10); |
| clone.text = new UnicodeUtil.UTF16Result(); |
| + clone.text.offsets = new int[text.offsets.length]; |
| + System.arraycopy(text.offsets, 0, clone.text.offsets, 0, text.offsets.length); |
| clone.text.copyText(text); |
| return clone; |
| } |
| Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (working copy) |
| @@ -39,11 +39,15 @@ |
| import org.apache.lucene.store.IndexInput; |
| import org.apache.lucene.util.Bits; |
| import org.apache.lucene.util.BytesRef; |
| +import org.apache.lucene.util.UnicodeUtil; |
| +import org.apache.lucene.util.ArrayUtil; |
| |
| /** Exposes flex API on a pre-flex index, as a codec. |
| * @lucene.experimental */ |
| public class PreFlexFields extends FieldsProducer { |
| |
| + private static final boolean DEBUG_SURROGATES = false; |
| + |
| public TermInfosReader tis; |
| public final TermInfosReader tisNoIndex; |
| |
| @@ -60,6 +64,16 @@ |
| throws IOException { |
| |
| si = info; |
| + |
| + // NOTE: we must always load terms index, even for |
| + // "sequential" scan during merging, because what is |
| + // sequential to merger may not be to TermInfosReader |
| + // since we do the surrogates dance: |
| + // nocommit -- how to pull right value from IW? |
| + if (indexDivisor < 0) { |
| + indexDivisor = -indexDivisor; |
| + } |
| + |
| TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, readBufferSize, indexDivisor); |
| if (indexDivisor == -1) { |
| tisNoIndex = r; |
| @@ -174,7 +188,6 @@ |
| private class PreFlexFieldsEnum extends FieldsEnum { |
| final Iterator<FieldInfo> it; |
| private final PreTermsEnum termsEnum; |
| - private int count; |
| FieldInfo current; |
| |
| public PreFlexFieldsEnum() throws IOException { |
| @@ -185,7 +198,6 @@ |
| @Override |
| public String next() { |
| if (it.hasNext()) { |
| - count++; |
| current = it.next(); |
| return current.name; |
| } else { |
| @@ -195,7 +207,7 @@ |
| |
| @Override |
| public TermsEnum terms() throws IOException { |
| - termsEnum.reset(current, count == 1); |
| + termsEnum.reset(current); |
| return termsEnum; |
| } |
| } |
| @@ -209,14 +221,15 @@ |
| @Override |
| public TermsEnum iterator() throws IOException { |
| PreTermsEnum termsEnum = new PreTermsEnum(); |
| - termsEnum.reset(fieldInfo, false); |
| + termsEnum.reset(fieldInfo); |
| return termsEnum; |
| } |
| |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - // Pre-flex indexes always sorted in UTF16 order |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + // Pre-flex indexes always sorted in UTF16 order, but |
| + // we remap on-the-fly to unicode order |
| + return BytesRef.getUTF8SortedAsUnicodeComparator(); |
| } |
| } |
| |
| @@ -227,37 +240,229 @@ |
| private BytesRef current; |
| private final BytesRef scratchBytesRef = new BytesRef(); |
| |
| - void reset(FieldInfo fieldInfo, boolean isFirstField) throws IOException { |
| + private int[] surrogateSeekPending = new int[1]; |
| + private boolean[] surrogateDidSeekBack = new boolean[1]; |
| + private int surrogateSeekUpto; |
| + private char[] pendingPrefix; |
| + |
| + private SegmentTermEnum seekTermEnum; |
| + private Term protoTerm; |
| + private int newSuffixStart; |
| + |
| + void reset(FieldInfo fieldInfo) throws IOException { |
| this.fieldInfo = fieldInfo; |
| + protoTerm = new Term(fieldInfo.name); |
| if (termEnum == null) { |
| - // First time reset is called |
| - if (isFirstField) { |
| - termEnum = getTermsDict().terms(); |
| - skipNext = false; |
| - } else { |
| - termEnum = getTermsDict().terms(new Term(fieldInfo.name, "")); |
| - skipNext = true; |
| - } |
| + termEnum = getTermsDict().terms(protoTerm); |
| + seekTermEnum = getTermsDict().terms(protoTerm); |
| } else { |
| - final Term t = termEnum.term(); |
| - if (t != null && t.field() == fieldInfo.name) { |
| - // No need to seek -- we have already advanced onto |
| - // this field. We must be @ first term because |
| - // flex API will not advance this enum further, on |
| - // seeing a different field. |
| - } else { |
| - assert t == null || !t.field().equals(fieldInfo.name); // make sure field name is interned |
| - final TermInfosReader tis = getTermsDict(); |
| - tis.seekEnum(termEnum, new Term(fieldInfo.name, "")); |
| + getTermsDict().seekEnum(termEnum, protoTerm); |
| + } |
| + skipNext = true; |
| + |
| + surrogateSeekUpto = 0; |
| + newSuffixStart = 0; |
| + |
| + surrogatesDance(); |
| + } |
| + |
| + private void surrogatesDance() throws IOException { |
| + |
| + // Tricky: prior to 4.0, Lucene index sorted terms in |
| + // UTF16 order, but as of 4.0 we sort by Unicode code |
| + // point order. These orders differ because of the |
| + // surrrogates; so we have to fixup our enum, here, by |
| + // carefully first seeking past the surrogates and |
| + // then back again at the end. The process is |
| + // recursive, since any given term could have multiple |
| + // new occurrences of surrogate pairs, so we use a |
| + // stack to record the pending seek-backs. |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" dance start term=" + (termEnum.term() == null ? null : UnicodeUtil.toHexString(termEnum.term().text()))); |
| + } |
| + |
| + while(popPendingSeek()); |
| + while(pushNewSurrogate()); |
| + } |
| + |
| + // only for debugging |
| + private String getStack() { |
| + if (surrogateSeekUpto == 0) { |
| + return "null"; |
| + } else { |
| + StringBuffer sb = new StringBuffer(); |
| + for(int i=0;i<surrogateSeekUpto;i++) { |
| + if (i > 0) { |
| + sb.append(' '); |
| + } |
| + sb.append(surrogateSeekPending[i]); |
| } |
| - skipNext = true; |
| + sb.append(" pendingSeekText=" + new String(pendingPrefix, 0, surrogateSeekPending[surrogateSeekUpto-1])); |
| + return sb.toString(); |
| } |
| } |
| |
| + private boolean popPendingSeek() throws IOException { |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" check pop newSuffix=" + newSuffixStart + " stack=" + getStack()); |
| + } |
| + // if a .next() has advanced beyond the |
| + // after-surrogates range we had last seeked to, we |
| + // must seek back to the start and resume .next from |
| + // there. this pops the pending seek off the stack. |
| + final Term t = termEnum.term(); |
| + if (surrogateSeekUpto > 0) { |
| + final int seekPrefix = surrogateSeekPending[surrogateSeekUpto-1]; |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" seekPrefix=" + seekPrefix); |
| + } |
| + if (newSuffixStart < seekPrefix) { |
| + assert pendingPrefix != null; |
| + assert pendingPrefix.length > seekPrefix; |
| + pendingPrefix[seekPrefix] = UnicodeUtil.UNI_SUR_HIGH_START; |
| + Term t2 = protoTerm.createTerm(new String(pendingPrefix, 0, 1+seekPrefix)); |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" do pop; seek back to " + UnicodeUtil.toHexString(t2.text())); |
| + } |
| + getTermsDict().seekEnum(termEnum, t2); |
| + surrogateDidSeekBack[surrogateSeekUpto-1] = true; |
| + |
| + // +2 because we don't want to re-check the |
| + // surrogates we just seek'd back to |
| + newSuffixStart = seekPrefix + 2; |
| + return true; |
| + } else if (newSuffixStart == seekPrefix && surrogateDidSeekBack[surrogateSeekUpto-1] && t != null && t.field() == fieldInfo.name && t.text().charAt(seekPrefix) > UnicodeUtil.UNI_SUR_LOW_END) { |
| + assert pendingPrefix != null; |
| + assert pendingPrefix.length > seekPrefix; |
| + pendingPrefix[seekPrefix] = 0xffff; |
| + Term t2 = protoTerm.createTerm(new String(pendingPrefix, 0, 1+seekPrefix)); |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" finish pop; seek fwd to " + UnicodeUtil.toHexString(t2.text())); |
| + } |
| + getTermsDict().seekEnum(termEnum, t2); |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" found term=" + (termEnum.term() == null ? null : UnicodeUtil.toHexString(termEnum.term().text()))); |
| + } |
| + surrogateSeekUpto--; |
| + |
| + if (termEnum.term() == null || termEnum.term().field() != fieldInfo.name) { |
| + // force pop |
| + newSuffixStart = -1; |
| + } else { |
| + newSuffixStart = termEnum.newSuffixStart; |
| + } |
| + |
| + return true; |
| + } |
| + } |
| + |
| + return false; |
| + } |
| + |
| + private boolean pushNewSurrogate() throws IOException { |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" check push newSuffix=" + newSuffixStart + " stack=" + getStack()); |
| + } |
| + final Term t = termEnum.term(); |
| + if (t == null || t.field() != fieldInfo.name) { |
| + return false; |
| + } |
| + final String text = t.text(); |
| + final int textLen = text.length(); |
| + |
| + for(int i=Math.max(0,newSuffixStart);i<textLen;i++) { |
| + final char ch = text.charAt(i); |
| + if (ch >= UnicodeUtil.UNI_SUR_HIGH_START && ch <= UnicodeUtil.UNI_SUR_HIGH_END && (surrogateSeekUpto == 0 || i > surrogateSeekPending[surrogateSeekUpto-1])) { |
| + |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" found high surr 0x" + Integer.toHexString(ch) + " at pos=" + i); |
| + } |
| + |
| + // the next() that we just did read in a new |
| + // suffix, containing a surrogate pair |
| + |
| + // seek forward to see if there are any terms with |
| + // this same prefix, but with characters after the |
| + // surrogate range; if so, we must first iterate |
| + // them, then seek back to the surrogates |
| + |
| + char[] testPrefix = new char[i+1]; |
| + for(int j=0;j<i;j++) { |
| + testPrefix[j] = text.charAt(j); |
| + } |
| + testPrefix[i] = 1+UnicodeUtil.UNI_SUR_LOW_END; |
| + |
| + getTermsDict().seekEnum(seekTermEnum, protoTerm.createTerm(new String(testPrefix))); |
| + |
| + Term t2 = seekTermEnum.term(); |
| + boolean isPrefix; |
| + if (t2 != null && t2.field() == fieldInfo.name) { |
| + String seekText = t2.text(); |
| + isPrefix = true; |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" seek found " + UnicodeUtil.toHexString(seekText)); |
| + } |
| + for(int j=0;j<i;j++) { |
| + if (testPrefix[j] != seekText.charAt(j)) { |
| + isPrefix = false; |
| + break; |
| + } |
| + } |
| + if (DEBUG_SURROGATES && !isPrefix) { |
| + System.out.println(" no end terms"); |
| + } |
| + } else { |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" no end terms"); |
| + } |
| + isPrefix = false; |
| + } |
| + |
| + if (isPrefix) { |
| + // we found a term, sharing the same prefix, |
| + // with characters after the surrogates, so we |
| + // must first enum those, and then return the |
| + // the surrogates afterwards. push that pending |
| + // seek on the surrogates stack now: |
| + pendingPrefix = testPrefix; |
| + |
| + getTermsDict().seekEnum(termEnum, t2); |
| + |
| + if (surrogateSeekUpto == surrogateSeekPending.length) { |
| + surrogateSeekPending = ArrayUtil.grow(surrogateSeekPending); |
| + } |
| + if (surrogateSeekUpto == surrogateDidSeekBack.length) { |
| + surrogateDidSeekBack = ArrayUtil.grow(surrogateDidSeekBack); |
| + } |
| + surrogateSeekPending[surrogateSeekUpto] = i; |
| + surrogateDidSeekBack[surrogateSeekUpto] = false; |
| + surrogateSeekUpto++; |
| + |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" do push " + i + "; end term=" + UnicodeUtil.toHexString(t2.text())); |
| + } |
| + |
| + newSuffixStart = i+1; |
| + |
| + return true; |
| + } else { |
| + // there are no terms after the surrogates, so |
| + // we do nothing to the enum and just step |
| + // through the surrogates like normal. but we |
| + // must keep iterating through the term, in case |
| + // another surrogate pair appears later |
| + } |
| + } |
| + } |
| + |
| + return false; |
| + } |
| + |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - // Pre-flex indexes always sorted in UTF16 order |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + // Pre-flex indexes always sorted in UTF16 order, but |
| + // we remap on-the-fly to unicode order |
| + return BytesRef.getUTF8SortedAsUnicodeComparator(); |
| } |
| |
| @Override |
| @@ -272,14 +477,24 @@ |
| |
| @Override |
| public SeekStatus seek(BytesRef term, boolean useCache) throws IOException { |
| + if (DEBUG_SURROGATES) { |
| + System.out.println("TE.seek() term=" + term.utf8ToString()); |
| + } |
| skipNext = false; |
| final TermInfosReader tis = getTermsDict(); |
| - final Term t0 = new Term(fieldInfo.name, term.utf8ToString()); |
| + final Term t0 = protoTerm.createTerm(term.utf8ToString()); |
| + |
| + assert termEnum != null; |
| + |
| if (termEnum == null) { |
| termEnum = tis.terms(t0); |
| } else { |
| tis.seekEnum(termEnum, t0); |
| } |
| + |
| + surrogateSeekUpto = 0; |
| + surrogatesDance(); |
| + |
| final Term t = termEnum.term(); |
| |
| final BytesRef tr; |
| @@ -304,6 +519,9 @@ |
| |
| @Override |
| public BytesRef next() throws IOException { |
| + if (DEBUG_SURROGATES) { |
| + System.out.println("TE.next() skipNext=" + skipNext); |
| + } |
| if (skipNext) { |
| skipNext = false; |
| if (termEnum.term() == null) { |
| @@ -313,19 +531,37 @@ |
| return current = scratchBytesRef; |
| } |
| } |
| - if (termEnum.next()) { |
| + if (termEnum.next() && termEnum.term().field() == fieldInfo.name) { |
| + newSuffixStart = termEnum.newSuffixStart; |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" set newSuffixStart=" + newSuffixStart); |
| + } |
| + surrogatesDance(); |
| final Term t = termEnum.term(); |
| - if (t.field() == fieldInfo.name) { |
| + if (t == null || t.field() != fieldInfo.name) { |
| + assert t == null || !t.field().equals(fieldInfo.name); // make sure fields are in fact interned |
| + current = null; |
| + } else { |
| scratchBytesRef.copy(t.text()); |
| current = scratchBytesRef; |
| + } |
| + return current; |
| + } else { |
| + if (DEBUG_SURROGATES) { |
| + System.out.println(" force pop"); |
| + } |
| + // force pop |
| + newSuffixStart = -1; |
| + surrogatesDance(); |
| + final Term t = termEnum.term(); |
| + if (t == null || t.field() != fieldInfo.name) { |
| + assert t == null || !t.field().equals(fieldInfo.name); // make sure fields are in fact interned |
| + return null; |
| + } else { |
| + scratchBytesRef.copy(t.text()); |
| + current = scratchBytesRef; |
| return current; |
| - } else { |
| - assert !t.field().equals(fieldInfo.name); // make sure field name is interned |
| - // Crossed into new field |
| - return null; |
| } |
| - } else { |
| - return null; |
| } |
| } |
| |
| Index: lucene/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java (working copy) |
| @@ -67,7 +67,7 @@ |
| |
| success = false; |
| try { |
| - FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUnicodeComparator()); |
| success = true; |
| return ret; |
| } finally { |
| @@ -95,7 +95,7 @@ |
| state.fieldInfos, |
| state.segmentInfo.name, |
| state.termsIndexDivisor, |
| - BytesRef.getUTF8SortedAsUTF16Comparator()); |
| + BytesRef.getUTF8SortedAsUnicodeComparator()); |
| success = true; |
| } finally { |
| if (!success) { |
| @@ -111,7 +111,7 @@ |
| state.segmentInfo.name, |
| postingsReader, |
| state.readBufferSize, |
| - BytesRef.getUTF8SortedAsUTF16Comparator(), |
| + BytesRef.getUTF8SortedAsUnicodeComparator(), |
| StandardCodec.TERMS_CACHE_SIZE); |
| success = true; |
| return ret; |
| Index: lucene/src/java/org/apache/lucene/util/automaton/Transition.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/util/automaton/Transition.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/util/automaton/Transition.java (working copy) |
| @@ -210,64 +210,4 @@ |
| } |
| |
| public static final Comparator<Transition> CompareByMinMaxThenDest = new CompareByMinMaxThenDestSingle(); |
| - |
| - private static class UTF8InUTF16Order { |
| - protected int compareCodePoint(int aByte, int bByte) { |
| - if (aByte != bByte) { |
| - // See http://icu-project.org/docs/papers/utf16_code_point_order.html#utf-8-in-utf-16-order |
| - |
| - // We know the terms are not equal, but, we may |
| - // have to carefully fixup the bytes at the |
| - // difference to match UTF16's sort order: |
| - if (aByte >= 0xee && bByte >= 0xee) { |
| - if ((aByte & 0xfe) == 0xee) { |
| - aByte += 0x10; |
| - } |
| - if ((bByte&0xfe) == 0xee) { |
| - bByte += 0x10; |
| - } |
| - } |
| - return aByte - bByte; |
| - } |
| - return 0; |
| - } |
| - } |
| - |
| - private static final class CompareByDestThenMinMaxUTF8InUTF16OrderSingle extends UTF8InUTF16Order implements Comparator<Transition> { |
| - public int compare(Transition t1, Transition t2) { |
| - if (t1.to != t2.to) { |
| - if (t1.to == null) return -1; |
| - else if (t2.to == null) return 1; |
| - else if (t1.to.number < t2.to.number) return -1; |
| - else if (t1.to.number > t2.to.number) return 1; |
| - } |
| - int minComp = compareCodePoint(t1.min, t2.min); |
| - if (minComp != 0) return minComp; |
| - int maxComp = compareCodePoint(t1.max, t2.max); |
| - if (maxComp != 0) return maxComp; |
| - return 0; |
| - } |
| - } |
| - |
| - public static final Comparator<Transition> CompareByDestThenMinMaxUTF8InUTF16Order = new CompareByDestThenMinMaxUTF8InUTF16OrderSingle(); |
| - |
| - private static final class CompareByMinMaxThenDestUTF8InUTF16OrderSingle extends UTF8InUTF16Order implements Comparator<Transition> { |
| - public int compare(Transition t1, Transition t2) { |
| - int minComp = compareCodePoint(t1.min, t2.min); |
| - if (minComp != 0) return minComp; |
| - int maxComp = compareCodePoint(t1.max, t2.max); |
| - if (maxComp != 0) return maxComp; |
| - if (t1.to != t2.to) { |
| - if (t1.to == null) return -1; |
| - else if (t2.to == null) return 1; |
| - else if (t1.to.number < t2.to.number) return -1; |
| - else if (t1.to.number > t2.to.number) return 1; |
| - } |
| - return 0; |
| - } |
| - } |
| - |
| - public static final Comparator<Transition> CompareByMinMaxThenDestUTF8InUTF16Order = new CompareByMinMaxThenDestUTF8InUTF16OrderSingle(); |
| - |
| - |
| } |
| Index: lucene/src/java/org/apache/lucene/util/ArrayUtil.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/util/ArrayUtil.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/util/ArrayUtil.java (working copy) |
| @@ -327,6 +327,29 @@ |
| return array; |
| } |
| |
| + public static boolean[] grow(boolean[] array, int minSize) { |
| + if (array.length < minSize) { |
| + boolean[] newArray = new boolean[oversize(minSize, 1)]; |
| + System.arraycopy(array, 0, newArray, 0, array.length); |
| + return newArray; |
| + } else |
| + return array; |
| + } |
| + |
| + public static boolean[] grow(boolean[] array) { |
| + return grow(array, 1 + array.length); |
| + } |
| + |
| + public static boolean[] shrink(boolean[] array, int targetSize) { |
| + final int newSize = getShrinkSize(array.length, targetSize, 1); |
| + if (newSize != array.length) { |
| + boolean[] newArray = new boolean[newSize]; |
| + System.arraycopy(array, 0, newArray, 0, newSize); |
| + return newArray; |
| + } else |
| + return array; |
| + } |
| + |
| public static char[] grow(char[] array, int minSize) { |
| if (array.length < minSize) { |
| char[] newArray = new char[oversize(minSize, RamUsageEstimator.NUM_BYTES_CHAR)]; |
| Index: lucene/src/java/org/apache/lucene/util/BytesRef.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/util/BytesRef.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/util/BytesRef.java (working copy) |
| @@ -217,14 +217,7 @@ |
| bytes = ArrayUtil.grow(bytes, newLength); |
| } |
| |
| - private final static Comparator<BytesRef> utf8SortedAsUTF16SortOrder = new UTF8SortedAsUTF16Comparator(); |
| - |
| - public static Comparator<BytesRef> getUTF8SortedAsUTF16Comparator() { |
| - return utf8SortedAsUTF16SortOrder; |
| - } |
| - |
| /** Unsigned byte order comparison */ |
| - /* |
| public int compareTo(BytesRef other) { |
| if (this == other) return 0; |
| |
| @@ -245,52 +238,18 @@ |
| // One is a prefix of the other, or, they are equal: |
| return this.length - other.length; |
| } |
| - */ |
| |
| - /** Lucene default index order. Currently the same as String.compareTo() (UTF16) but will change |
| - * in the future to unsigned byte comparison. */ |
| - public int compareTo(BytesRef other) { |
| - if (this == other) return 0; |
| + private final static Comparator<BytesRef> utf8SortedAsUnicodeSortOrder = new UTF8SortedAsUnicodeComparator(); |
| |
| - final byte[] aBytes = this.bytes; |
| - int aUpto = this.offset; |
| - final byte[] bBytes = other.bytes; |
| - int bUpto = other.offset; |
| - |
| - final int aStop = aUpto + Math.min(this.length, other.length); |
| - |
| - while(aUpto < aStop) { |
| - int aByte = aBytes[aUpto++] & 0xff; |
| - int bByte = bBytes[bUpto++] & 0xff; |
| - if (aByte != bByte) { |
| - |
| - // See http://icu-project.org/docs/papers/utf16_code_point_order.html#utf-8-in-utf-16-order |
| - |
| - // We know the terms are not equal, but, we may |
| - // have to carefully fixup the bytes at the |
| - // difference to match UTF16's sort order: |
| - if (aByte >= 0xee && bByte >= 0xee) { |
| - if ((aByte & 0xfe) == 0xee) { |
| - aByte += 0x10; |
| - } |
| - if ((bByte&0xfe) == 0xee) { |
| - bByte += 0x10; |
| - } |
| - } |
| - return aByte - bByte; |
| - } |
| - } |
| - |
| - // One is a prefix of the other, or, they are equal: |
| - return this.length - other.length; |
| + public static Comparator<BytesRef> getUTF8SortedAsUnicodeComparator() { |
| + return utf8SortedAsUnicodeSortOrder; |
| } |
| |
| - private static class UTF8SortedAsUTF16Comparator implements Comparator<BytesRef> { |
| + private static class UTF8SortedAsUnicodeComparator implements Comparator<BytesRef> { |
| // Only singleton |
| - private UTF8SortedAsUTF16Comparator() {}; |
| + private UTF8SortedAsUnicodeComparator() {}; |
| |
| public int compare(BytesRef a, BytesRef b) { |
| - |
| final byte[] aBytes = a.bytes; |
| int aUpto = a.offset; |
| final byte[] bBytes = b.bytes; |
| @@ -307,32 +266,15 @@ |
| int aByte = aBytes[aUpto++] & 0xff; |
| int bByte = bBytes[bUpto++] & 0xff; |
| |
| - if (aByte != bByte) { |
| - |
| - // See http://icu-project.org/docs/papers/utf16_code_point_order.html#utf-8-in-utf-16-order |
| - |
| - // We know the terms are not equal, but, we may |
| - // have to carefully fixup the bytes at the |
| - // difference to match UTF16's sort order: |
| - if (aByte >= 0xee && bByte >= 0xee) { |
| - if ((aByte & 0xfe) == 0xee) { |
| - aByte += 0x10; |
| - } |
| - if ((bByte&0xfe) == 0xee) { |
| - bByte += 0x10; |
| - } |
| - } |
| - return aByte - bByte; |
| + int diff = aByte - bByte; |
| + if (diff != 0) { |
| + return diff; |
| } |
| } |
| |
| // One is a prefix of the other, or, they are equal: |
| return a.length - b.length; |
| - } |
| - |
| - public boolean equals(Object other) { |
| - return this == other; |
| - } |
| + } |
| } |
| |
| public void writeExternal(ObjectOutput out) |
| Index: lucene/src/java/org/apache/lucene/util/UnicodeUtil.java |
| =================================================================== |
| --- lucene/src/java/org/apache/lucene/util/UnicodeUtil.java (revision 956375) |
| +++ lucene/src/java/org/apache/lucene/util/UnicodeUtil.java (working copy) |
| @@ -358,7 +358,6 @@ |
| out[outUpto++] = (char) ((chHalf & HALF_MASK) + UNI_SUR_LOW_START); |
| } |
| } |
| - |
| offsets[upto] = outUpto; |
| result.length = outUpto; |
| } |
| @@ -483,7 +482,7 @@ |
| } |
| } |
| */ |
| - public static final boolean validUTF16String(CharSequence s) { |
| + public static boolean validUTF16String(CharSequence s) { |
| final int size = s.length(); |
| for(int i=0;i<size;i++) { |
| char ch = s.charAt(i); |
| @@ -507,7 +506,7 @@ |
| return true; |
| } |
| |
| - public static final boolean validUTF16String(char[] s, int size) { |
| + public static boolean validUTF16String(char[] s, int size) { |
| for(int i=0;i<size;i++) { |
| char ch = s[i]; |
| if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { |
| @@ -559,7 +558,7 @@ |
| /** Returns the number of code points in this utf8 |
| * sequence. Behavior is undefined if the utf8 sequence |
| * is invalid.*/ |
| - public static final int codePointCount(BytesRef utf8) { |
| + public static int codePointCount(BytesRef utf8) { |
| int upto = utf8.offset; |
| final int limit = utf8.offset + utf8.length; |
| final byte[] bytes = utf8.bytes; |
| @@ -673,4 +672,33 @@ |
| } |
| return new String(chars, 0, w); |
| } |
| + |
| + // for debugging |
| + public static String toHexString(String s) { |
| + StringBuilder sb = new StringBuilder(); |
| + for(int i=0;i<s.length();i++) { |
| + char ch = s.charAt(i); |
| + if (i > 0) { |
| + sb.append(' '); |
| + } |
| + if (ch < 128) { |
| + sb.append(ch); |
| + } else { |
| + if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { |
| + sb.append("H:"); |
| + } else if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { |
| + sb.append("L:"); |
| + } else if (ch > UNI_SUR_LOW_END) { |
| + if (ch == 0xffff) { |
| + sb.append("F:"); |
| + } else { |
| + sb.append("E:"); |
| + } |
| + } |
| + |
| + sb.append("0x" + Integer.toHexString(ch)); |
| + } |
| + } |
| + return sb.toString(); |
| + } |
| } |
| Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java |
| =================================================================== |
| --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (revision 956375) |
| +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexReader.java (working copy) |
| @@ -426,7 +426,7 @@ |
| |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + return BytesRef.getUTF8SortedAsUnicodeComparator(); |
| } |
| }; |
| } |
| Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java |
| =================================================================== |
| --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java (revision 956375) |
| +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java (working copy) |
| @@ -123,7 +123,7 @@ |
| |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + return BytesRef.getUTF8SortedAsUnicodeComparator(); |
| } |
| } |
| |
| Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java |
| =================================================================== |
| --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 956375) |
| +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) |
| @@ -808,7 +808,7 @@ |
| |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + return BytesRef.getUTF8SortedAsUnicodeComparator(); |
| } |
| |
| @Override |
| @@ -903,7 +903,7 @@ |
| |
| @Override |
| public Comparator<BytesRef> getComparator() { |
| - return BytesRef.getUTF8SortedAsUTF16Comparator(); |
| + return BytesRef.getUTF8SortedAsUnicodeComparator(); |
| } |
| } |
| |
