| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.lucene.index; |
| |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.List; |
| import java.util.Random; |
| import java.util.concurrent.CountDownLatch; |
| import java.util.function.LongSupplier; |
| |
| import org.apache.lucene.analysis.Analyzer; |
| import org.apache.lucene.analysis.MockAnalyzer; |
| import org.apache.lucene.analysis.MockTokenizer; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.Field; |
| import org.apache.lucene.document.NumericDocValuesField; |
| import org.apache.lucene.document.StringField; |
| import org.apache.lucene.document.TextField; |
| import org.apache.lucene.search.CollectionStatistics; |
| import org.apache.lucene.search.DocIdSetIterator; |
| import org.apache.lucene.search.TermStatistics; |
| import org.apache.lucene.search.similarities.Similarity; |
| import org.apache.lucene.store.Directory; |
| import org.apache.lucene.util.FixedBitSet; |
| import org.apache.lucene.util.IOUtils; |
| import org.apache.lucene.util.TestUtil; |
| |
| import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; |
| |
| /** |
| * Abstract class to do basic tests for a norms format. |
| * NOTE: This test focuses on the norms impl, nothing else. |
| * The [stretch] goal is for this test to be |
| * so thorough in testing a new NormsFormat that if this |
| * test passes, then all Lucene/Solr tests should also pass. Ie, |
| * if there is some bug in a given NormsFormat that this |
| * test fails to catch then this test needs to be improved! */ |
| public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCase { |
| |
| /** Whether the codec supports sparse values. */ |
| protected boolean codecSupportsSparsity() { |
| return true; |
| } |
| |
| public void testByteRange() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| }); |
| } |
| } |
| |
| public void testSparseByteRange() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| }); |
| } |
| } |
| |
| public void testShortRange() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return TestUtil.nextLong(r, Short.MIN_VALUE, Short.MAX_VALUE); |
| } |
| }); |
| } |
| } |
| |
| public void testSparseShortRange() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return TestUtil.nextLong(r, Short.MIN_VALUE, Short.MAX_VALUE); |
| } |
| }); |
| } |
| } |
| |
| public void testLongRange() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE); |
| } |
| }); |
| } |
| } |
| |
| public void testSparseLongRange() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE); |
| } |
| }); |
| } |
| } |
| |
| public void testFullLongRange() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| int thingToDo = r.nextInt(3); |
| switch (thingToDo) { |
| case 0: return Long.MIN_VALUE; |
| case 1: return Long.MAX_VALUE; |
| default: return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE); |
| } |
| } |
| }); |
| } |
| } |
| |
| public void testSparseFullLongRange() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| int thingToDo = r.nextInt(3); |
| switch (thingToDo) { |
| case 0: return Long.MIN_VALUE; |
| case 1: return Long.MAX_VALUE; |
| default: return TestUtil.nextLong(r, Long.MIN_VALUE, Long.MAX_VALUE); |
| } |
| } |
| }); |
| } |
| } |
| |
| public void testFewValues() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextBoolean() ? 20 : 3; |
| } |
| }); |
| } |
| } |
| |
| public void testFewSparseValues() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextBoolean() ? 20 : 3; |
| } |
| }); |
| } |
| } |
| |
| public void testFewLargeValues() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextBoolean() ? 1000000L : -5000; |
| } |
| }); |
| } |
| } |
| |
| public void testFewSparseLargeValues() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextBoolean() ? 1000000L : -5000; |
| } |
| }); |
| } |
| } |
| |
| public void testAllZeros() throws Exception { |
| int iterations = atLeast(1); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return 0; |
| } |
| }); |
| } |
| } |
| |
| public void testSparseAllZeros() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return 0; |
| } |
| }); |
| } |
| } |
| |
| public void testMostZeros() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE) : 0; |
| } |
| }); |
| } |
| } |
| |
| public void testOutliers() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE) : commonValue; |
| } |
| }); |
| } |
| } |
| |
| public void testSparseOutliers() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE) : commonValue; |
| } |
| }); |
| } |
| } |
| |
| public void testOutliers2() throws Exception { |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| final long uncommonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? uncommonValue : commonValue; |
| } |
| }); |
| } |
| } |
| |
| public void testSparseOutliers2() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; i++) { |
| final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| final long uncommonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? uncommonValue : commonValue; |
| } |
| }); |
| } |
| } |
| |
| public void testNCommon() throws Exception { |
| final Random r = random(); |
| final int N = TestUtil.nextInt(r, 2, 15); |
| final long[] commonValues = new long[N]; |
| for (int j = 0; j < N; ++j) { |
| commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N); |
| final long[] otherValues = new long[numOtherValues]; |
| for (int j = 0; j < numOtherValues; ++j) { |
| otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? otherValues[r.nextInt(numOtherValues - 1)] : commonValues[r.nextInt(N - 1)]; |
| } |
| }); |
| } |
| |
| public void testSparseNCommon() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| final Random r = random(); |
| final int N = TestUtil.nextInt(r, 2, 15); |
| final long[] commonValues = new long[N]; |
| for (int j = 0; j < N; ++j) { |
| commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N); |
| final long[] otherValues = new long[numOtherValues]; |
| for (int j = 0; j < numOtherValues; ++j) { |
| otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? otherValues[r.nextInt(numOtherValues - 1)] : commonValues[r.nextInt(N - 1)]; |
| } |
| }); |
| } |
| |
| /** |
| * a more thorough n-common that tests all low bpv |
| */ |
| @Nightly |
| public void testNCommonBig() throws Exception { |
| final int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; ++i) { |
| // 16 is 4 bpv, the max before we jump to 8bpv |
| for (int n = 2; n < 16; ++n) { |
| final int N = n; |
| final long[] commonValues = new long[N]; |
| for (int j = 0; j < N; ++j) { |
| commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N); |
| final long[] otherValues = new long[numOtherValues]; |
| for (int j = 0; j < numOtherValues; ++j) { |
| otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| doTestNormsVersusDocValues(1, new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? otherValues[r.nextInt(numOtherValues - 1)] : commonValues[r.nextInt(N - 1)]; |
| } |
| }); |
| } |
| } |
| } |
| |
| /** |
| * a more thorough n-common that tests all low bpv and sparse docs |
| */ |
| @Nightly |
| public void testSparseNCommonBig() throws Exception { |
| assumeTrue("Requires sparse norms support", codecSupportsSparsity()); |
| final int iterations = atLeast(1); |
| final Random r = random(); |
| for (int i = 0; i < iterations; ++i) { |
| // 16 is 4 bpv, the max before we jump to 8bpv |
| for (int n = 2; n < 16; ++n) { |
| final int N = n; |
| final long[] commonValues = new long[N]; |
| for (int j = 0; j < N; ++j) { |
| commonValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| final int numOtherValues = TestUtil.nextInt(r, 2, 256 - N); |
| final long[] otherValues = new long[numOtherValues]; |
| for (int j = 0; j < numOtherValues; ++j) { |
| otherValues[j] = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE); |
| } |
| doTestNormsVersusDocValues(random().nextDouble(), new LongSupplier() { |
| @Override |
| public long getAsLong() { |
| return r.nextInt(100) == 0 ? otherValues[r.nextInt(numOtherValues - 1)] : commonValues[r.nextInt(N - 1)]; |
| } |
| }); |
| } |
| } |
| } |
| |
| private void doTestNormsVersusDocValues(double density, LongSupplier longs) throws Exception { |
| int numDocs = atLeast(500); |
| final FixedBitSet docsWithField = new FixedBitSet(numDocs); |
| final int numDocsWithField = Math.max(1, (int) (density * numDocs)); |
| if (numDocsWithField == numDocs) { |
| docsWithField.set(0, numDocs); |
| } else { |
| int i = 0; |
| while (i < numDocsWithField) { |
| int doc = random().nextInt(numDocs); |
| if (docsWithField.get(doc) == false) { |
| docsWithField.set(doc); |
| ++i; |
| } |
| } |
| } |
| long norms[] = new long[numDocsWithField]; |
| for (int i = 0; i < numDocsWithField; i++) { |
| norms[i] = longs.getAsLong(); |
| } |
| |
| Directory dir = applyCreatedVersionMajor(newDirectory()); |
| Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); |
| IndexWriterConfig conf = newIndexWriterConfig(analyzer); |
| CannedNormSimilarity sim = new CannedNormSimilarity(norms); |
| conf.setSimilarity(sim); |
| RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf); |
| Document doc = new Document(); |
| Field idField = new StringField("id", "", Field.Store.NO); |
| Field indexedField = new TextField("indexed", "", Field.Store.NO); |
| Field dvField = new NumericDocValuesField("dv", 0); |
| doc.add(idField); |
| doc.add(indexedField); |
| doc.add(dvField); |
| |
| for (int i = 0, j = 0; i < numDocs; i++) { |
| idField.setStringValue(Integer.toString(i)); |
| if (docsWithField.get(i) == false) { |
| Document doc2 = new Document(); |
| doc2.add(idField); |
| writer.addDocument(doc2); |
| } else { |
| long value = norms[j++]; |
| dvField.setLongValue(value); |
| // only empty fields may have 0 as a norm |
| indexedField.setStringValue(value == 0 ? "" : "a"); |
| writer.addDocument(doc); |
| } |
| if (random().nextInt(31) == 0) { |
| writer.commit(); |
| } |
| } |
| |
| // delete some docs |
| int numDeletions = random().nextInt(numDocs/20); |
| for (int i = 0; i < numDeletions; i++) { |
| int id = random().nextInt(numDocs); |
| writer.deleteDocuments(new Term("id", Integer.toString(id))); |
| } |
| |
| writer.commit(); |
| |
| // compare |
| DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir)); |
| checkNormsVsDocValues(ir); |
| ir.close(); |
| |
| writer.forceMerge(1); |
| |
| // compare again |
| ir = maybeWrapWithMergingReader(DirectoryReader.open(dir)); |
| checkNormsVsDocValues(ir); |
| |
| writer.close(); |
| ir.close(); |
| dir.close(); |
| } |
| |
| private void checkNormsVsDocValues(IndexReader ir) throws IOException { |
| for (LeafReaderContext context : ir.leaves()) { |
| LeafReader r = context.reader(); |
| NumericDocValues expected = r.getNumericDocValues("dv"); |
| NumericDocValues actual = r.getNormValues("indexed"); |
| assertEquals(expected == null, actual == null); |
| if (expected != null) { |
| for (int d = expected.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = expected.nextDoc()) { |
| assertEquals(d, actual.nextDoc()); |
| assertEquals("doc " + d, expected.longValue(), actual.longValue()); |
| } |
| assertEquals(NO_MORE_DOCS, actual.nextDoc()); |
| } |
| } |
| } |
| |
| static class CannedNormSimilarity extends Similarity { |
| final long norms[]; |
| int index = 0; |
| |
| CannedNormSimilarity(long norms[]) { |
| this.norms = norms; |
| } |
| |
| @Override |
| public long computeNorm(FieldInvertState state) { |
| assert state.length > 0; |
| while (true) { |
| long norm = norms[index++]; |
| if (norm != 0) { |
| return norm; |
| } |
| } |
| } |
| |
| @Override |
| public SimScorer scorer(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) { |
| throw new UnsupportedOperationException(); |
| } |
| } |
| |
| @Override |
| protected void addRandomFields(Document doc) { |
| // TODO: improve |
| doc.add(new TextField("foobar", TestUtil.randomSimpleString(random()), Field.Store.NO)); |
| |
| } |
| |
| @Override |
| public void testMergeStability() throws Exception { |
| // TODO: can we improve this base test to just have subclasses declare the extensions to check, |
| // rather than a blacklist to exclude? we need to index stuff to get norms, but we dont care about testing |
| // the PFs actually doing that... |
| assumeTrue("The MockRandom PF randomizes content on the fly, so we can't check it", false); |
| } |
| |
| // TODO: test thread safety (e.g. across different fields) explicitly here |
| |
| /* |
| * LUCENE-6006: Tests undead norms. |
| * ..... |
| * C C / |
| * /< / |
| * ___ __________/_#__=o |
| * /(- /(\_\________ \ |
| * \ ) \ )_ \o \ |
| * /|\ /|\ |' | |
| * | _| |
| * /o __\ |
| * / ' | |
| * / / | |
| * /_/\______| |
| * ( _( < |
| * \ \ \ |
| * \ \ | |
| * \____\____\ |
| * ____\_\__\_\ |
| * /` /` o\ |
| * |___ |_______| |
| * |
| */ |
| public void testUndeadNorms() throws Exception { |
| Directory dir = applyCreatedVersionMajor(newDirectory()); |
| RandomIndexWriter w = new RandomIndexWriter(random(), dir); |
| int numDocs = atLeast(500); |
| List<Integer> toDelete = new ArrayList<>(); |
| for(int i=0;i<numDocs;i++) { |
| Document doc = new Document(); |
| doc.add(new StringField("id", ""+i, Field.Store.NO)); |
| if (random().nextInt(5) == 1) { |
| toDelete.add(i); |
| doc.add(new TextField("content", "some content", Field.Store.NO)); |
| } |
| w.addDocument(doc); |
| } |
| for(Integer id : toDelete) { |
| w.deleteDocuments(new Term("id", ""+id)); |
| } |
| w.forceMerge(1); |
| IndexReader r = maybeWrapWithMergingReader(w.getReader()); |
| assertFalse(r.hasDeletions()); |
| |
| // Confusingly, norms should exist, and should all be 0, even though we deleted all docs that had the field "content". They should not |
| // be undead: |
| NumericDocValues norms = MultiDocValues.getNormValues(r, "content"); |
| assertNotNull(norms); |
| if (codecSupportsSparsity()) { |
| assertEquals(DocIdSetIterator.NO_MORE_DOCS, norms.nextDoc()); |
| } else { |
| for(int i=0;i<r.maxDoc();i++) { |
| assertEquals(i, norms.nextDoc()); |
| assertEquals(0, norms.longValue()); |
| } |
| } |
| |
| r.close(); |
| w.close(); |
| dir.close(); |
| } |
| |
| public void testThreads() throws Exception { |
| float density = codecSupportsSparsity() == false || random().nextBoolean() ? 1f : random().nextFloat(); |
| int numDocs = atLeast(500); |
| final FixedBitSet docsWithField = new FixedBitSet(numDocs); |
| final int numDocsWithField = Math.max(1, (int) (density * numDocs)); |
| if (numDocsWithField == numDocs) { |
| docsWithField.set(0, numDocs); |
| } else { |
| int i = 0; |
| while (i < numDocsWithField) { |
| int doc = random().nextInt(numDocs); |
| if (docsWithField.get(doc) == false) { |
| docsWithField.set(doc); |
| ++i; |
| } |
| } |
| } |
| |
| long norms[] = new long[numDocsWithField]; |
| for (int i = 0; i < numDocsWithField; i++) { |
| norms[i] = random().nextLong(); |
| } |
| |
| Directory dir = applyCreatedVersionMajor(newDirectory()); |
| Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); |
| IndexWriterConfig conf = newIndexWriterConfig(analyzer);conf.setMergePolicy(NoMergePolicy.INSTANCE); |
| conf.setSimilarity(new CannedNormSimilarity(norms)); |
| RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf); |
| Document doc = new Document(); |
| Field idField = new StringField("id", "", Field.Store.NO); |
| Field indexedField = new TextField("indexed", "", Field.Store.NO); |
| Field dvField = new NumericDocValuesField("dv", 0); |
| doc.add(idField); |
| doc.add(indexedField); |
| doc.add(dvField); |
| |
| for (int i = 0, j = 0; i < numDocs; i++) { |
| idField.setStringValue(Integer.toString(i)); |
| if (docsWithField.get(i) == false) { |
| Document doc2 = new Document(); |
| doc2.add(idField); |
| writer.addDocument(doc2); |
| } else { |
| long value = norms[j++]; |
| dvField.setLongValue(value); |
| indexedField.setStringValue(value == 0 ? "" : "a"); |
| writer.addDocument(doc); |
| } |
| if (random().nextInt(31) == 0) { |
| writer.commit(); |
| } |
| } |
| |
| DirectoryReader reader = maybeWrapWithMergingReader(writer.getReader()); |
| writer.close(); |
| |
| final int numThreads = TestUtil.nextInt(random(), 3, 30); |
| Thread[] threads = new Thread[numThreads]; |
| final CountDownLatch latch = new CountDownLatch(1); |
| for (int i = 0; i < numThreads; ++i) { |
| threads[i] = new Thread(new Runnable() { |
| @Override |
| public void run() { |
| try { |
| latch.await(); |
| checkNormsVsDocValues(reader); |
| TestUtil.checkReader(reader); |
| } catch (Exception e) { |
| throw new RuntimeException(e); |
| } |
| } |
| }); |
| } |
| |
| for (Thread thread : threads) { |
| thread.start(); |
| } |
| latch.countDown(); |
| for (Thread thread : threads) { |
| thread.join(); |
| } |
| |
| reader.close(); |
| dir.close(); |
| } |
| |
| public void testIndependantIterators() throws IOException { |
| Directory dir = newDirectory(); |
| IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy()); |
| CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20}); |
| conf.setSimilarity(sim); |
| RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf); |
| Document doc = new Document(); |
| Field indexedField = new TextField("indexed", "a", Field.Store.NO); |
| doc.add(indexedField); |
| for (int i = 0; i < 3; ++i) { |
| writer.addDocument(doc); |
| } |
| writer.forceMerge(1); |
| LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader())); |
| NumericDocValues n1 = r.getNormValues("indexed"); |
| NumericDocValues n2 = r.getNormValues("indexed"); |
| assertEquals(0, n1.nextDoc()); |
| assertEquals(42, n1.longValue()); |
| assertEquals(1, n1.nextDoc()); |
| assertEquals(10, n1.longValue()); |
| assertEquals(0, n2.nextDoc()); |
| assertEquals(42, n2.longValue()); |
| assertEquals(1, n2.nextDoc()); |
| assertEquals(10, n2.longValue()); |
| assertEquals(2, n2.nextDoc()); |
| assertEquals(20, n2.longValue()); |
| assertEquals(2, n1.nextDoc()); |
| assertEquals(20, n1.longValue()); |
| assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc()); |
| assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc()); |
| IOUtils.close(r, writer, dir); |
| } |
| |
| public void testIndependantSparseIterators() throws IOException { |
| Directory dir = newDirectory(); |
| IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy()); |
| CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20}); |
| conf.setSimilarity(sim); |
| RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf); |
| Document doc = new Document(); |
| Field indexedField = new TextField("indexed", "a", Field.Store.NO); |
| doc.add(indexedField); |
| Document emptyDoc = new Document(); |
| for (int i = 0; i < 3; ++i) { |
| writer.addDocument(doc); |
| writer.addDocument(emptyDoc); |
| } |
| writer.forceMerge(1); |
| LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader())); |
| NumericDocValues n1 = r.getNormValues("indexed"); |
| NumericDocValues n2 = r.getNormValues("indexed"); |
| assertEquals(0, n1.nextDoc()); |
| assertEquals(42, n1.longValue()); |
| assertEquals(2, n1.nextDoc()); |
| assertEquals(10, n1.longValue()); |
| assertEquals(0, n2.nextDoc()); |
| assertEquals(42, n2.longValue()); |
| assertEquals(2, n2.nextDoc()); |
| assertEquals(10, n2.longValue()); |
| assertEquals(4, n2.nextDoc()); |
| assertEquals(20, n2.longValue()); |
| assertEquals(4, n1.nextDoc()); |
| assertEquals(20, n1.longValue()); |
| assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc()); |
| assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc()); |
| IOUtils.close(r, writer, dir); |
| } |
| } |
