| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.lucene.index; |
| |
| import java.io.IOException; |
| import java.util.Collections; |
| import java.util.HashSet; |
| import java.util.Map; |
| import java.util.Random; |
| import java.util.Set; |
| |
| import org.apache.lucene.codecs.Codec; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.FieldType; |
| import org.apache.lucene.document.StoredField; |
| import org.apache.lucene.document.TextField; |
| import org.apache.lucene.store.Directory; |
| import org.apache.lucene.store.IOContext; |
| import org.apache.lucene.store.MockDirectoryWrapper; |
| import org.apache.lucene.store.MockDirectoryWrapper.Failure; |
| import org.apache.lucene.store.MockDirectoryWrapper.FakeIOException; |
| import org.apache.lucene.util.StringHelper; |
| import org.apache.lucene.util.TestUtil; |
| import org.apache.lucene.util.Version; |
| |
| /** |
| * Abstract class to do basic tests for fis format. |
| * NOTE: This test focuses on the fis impl, nothing else. |
| * The [stretch] goal is for this test to be |
| * so thorough in testing a new fis format that if this |
| * test passes, then all Lucene/Solr tests should also pass. Ie, |
| * if there is some bug in a given fis Format that this |
| * test fails to catch then this test needs to be improved! */ |
| public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTestCase { |
| |
| /** Test field infos read/write with a single field */ |
| public void testOneField() throws Exception { |
| Directory dir = newDirectory(); |
| Codec codec = getCodec(); |
| SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); |
| FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); |
| FieldInfo fi = builder.getOrAdd("field"); |
| fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); |
| addAttributes(fi); |
| FieldInfos infos = builder.finish(); |
| codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT); |
| FieldInfos infos2 = codec.fieldInfosFormat().read(dir, segmentInfo, "", IOContext.DEFAULT); |
| assertEquals(1, infos2.size()); |
| assertNotNull(infos2.fieldInfo("field")); |
| assertTrue(infos2.fieldInfo("field").getIndexOptions() != IndexOptions.NONE); |
| assertFalse(infos2.fieldInfo("field").getDocValuesType() != DocValuesType.NONE); |
| assertFalse(infos2.fieldInfo("field").omitsNorms()); |
| assertFalse(infos2.fieldInfo("field").hasPayloads()); |
| assertFalse(infos2.fieldInfo("field").hasVectors()); |
| assertEquals(0, infos2.fieldInfo("field").getPointDimensionCount()); |
| assertFalse(infos2.fieldInfo("field").isSoftDeletesField()); |
| dir.close(); |
| } |
| |
| /** Test field infos attributes coming back are not mutable */ |
| public void testImmutableAttributes() throws Exception { |
| Directory dir = newDirectory(); |
| Codec codec = getCodec(); |
| SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); |
| FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); |
| FieldInfo fi = builder.getOrAdd("field"); |
| fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); |
| addAttributes(fi); |
| fi.putAttribute("foo", "bar"); |
| fi.putAttribute("bar", "baz"); |
| FieldInfos infos = builder.finish(); |
| codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT); |
| FieldInfos infos2 = codec.fieldInfosFormat().read(dir, segmentInfo, "", IOContext.DEFAULT); |
| assertEquals(1, infos2.size()); |
| assertNotNull(infos2.fieldInfo("field")); |
| Map<String,String> attributes = infos2.fieldInfo("field").attributes(); |
| // shouldn't be able to modify attributes |
| expectThrows(UnsupportedOperationException.class, () -> { |
| attributes.put("bogus", "bogus"); |
| }); |
| |
| dir.close(); |
| } |
| |
| /** |
| * Test field infos write that hits exception immediately on open. |
| * make sure we get our exception back, no file handle leaks, etc. |
| */ |
| public void testExceptionOnCreateOutput() throws Exception { |
| Failure fail = new Failure() { |
| @Override |
| public void eval(MockDirectoryWrapper dir) throws IOException { |
| if (doFail && callStackContainsAnyOf("createOutput")) { |
| throw new FakeIOException(); |
| } |
| } |
| }; |
| |
| MockDirectoryWrapper dir = newMockDirectory(); |
| dir.failOn(fail); |
| Codec codec = getCodec(); |
| SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); |
| FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); |
| FieldInfo fi = builder.getOrAdd("field"); |
| fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); |
| addAttributes(fi); |
| FieldInfos infos = builder.finish(); |
| |
| fail.setDoFail(); |
| expectThrows(FakeIOException.class, () -> { |
| codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT); |
| }); |
| fail.clearDoFail(); |
| |
| dir.close(); |
| } |
| |
| /** |
| * Test field infos write that hits exception on close. |
| * make sure we get our exception back, no file handle leaks, etc. |
| */ |
| public void testExceptionOnCloseOutput() throws Exception { |
| Failure fail = new Failure() { |
| @Override |
| public void eval(MockDirectoryWrapper dir) throws IOException { |
| if (doFail && callStackContainsAnyOf("close")) { |
| throw new FakeIOException(); |
| } |
| } |
| }; |
| |
| MockDirectoryWrapper dir = newMockDirectory(); |
| dir.failOn(fail); |
| Codec codec = getCodec(); |
| SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); |
| FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); |
| FieldInfo fi = builder.getOrAdd("field"); |
| fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); |
| addAttributes(fi); |
| FieldInfos infos = builder.finish(); |
| |
| fail.setDoFail(); |
| expectThrows(FakeIOException.class, () -> { |
| codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT); |
| }); |
| fail.clearDoFail(); |
| |
| dir.close(); |
| } |
| |
| /** |
| * Test field infos read that hits exception immediately on open. |
| * make sure we get our exception back, no file handle leaks, etc. |
| */ |
| public void testExceptionOnOpenInput() throws Exception { |
| Failure fail = new Failure() { |
| @Override |
| public void eval(MockDirectoryWrapper dir) throws IOException { |
| if (doFail && callStackContainsAnyOf("openInput")) { |
| throw new FakeIOException(); |
| } |
| } |
| }; |
| |
| MockDirectoryWrapper dir = newMockDirectory(); |
| dir.failOn(fail); |
| Codec codec = getCodec(); |
| SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); |
| FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); |
| FieldInfo fi = builder.getOrAdd("field"); |
| fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); |
| addAttributes(fi); |
| FieldInfos infos = builder.finish(); |
| codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT); |
| |
| fail.setDoFail(); |
| expectThrows(FakeIOException.class, () -> { |
| codec.fieldInfosFormat().read(dir, segmentInfo, "", IOContext.DEFAULT); |
| }); |
| fail.clearDoFail(); |
| |
| dir.close(); |
| } |
| |
| /** |
| * Test field infos read that hits exception on close. |
| * make sure we get our exception back, no file handle leaks, etc. |
| */ |
| public void testExceptionOnCloseInput() throws Exception { |
| Failure fail = new Failure() { |
| @Override |
| public void eval(MockDirectoryWrapper dir) throws IOException { |
| if (doFail && callStackContainsAnyOf("close")) { |
| throw new FakeIOException(); |
| } |
| } |
| }; |
| |
| MockDirectoryWrapper dir = newMockDirectory(); |
| dir.failOn(fail); |
| Codec codec = getCodec(); |
| SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); |
| FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null)); |
| FieldInfo fi = builder.getOrAdd("field"); |
| fi.setIndexOptions(TextField.TYPE_STORED.indexOptions()); |
| addAttributes(fi); |
| FieldInfos infos = builder.finish(); |
| codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT); |
| |
| fail.setDoFail(); |
| expectThrows(FakeIOException.class, () -> { |
| codec.fieldInfosFormat().read(dir, segmentInfo, "", IOContext.DEFAULT); |
| }); |
| fail.clearDoFail(); |
| |
| dir.close(); |
| } |
| |
| // TODO: more tests |
| |
| /** Test field infos read/write with random fields, with different values. */ |
| public void testRandom() throws Exception { |
| Directory dir = newDirectory(); |
| Codec codec = getCodec(); |
| SegmentInfo segmentInfo = newSegmentInfo(dir, "_123"); |
| |
| // generate a bunch of fields |
| int numFields = atLeast(2000); |
| Set<String> fieldNames = new HashSet<>(); |
| for (int i = 0; i < numFields; i++) { |
| fieldNames.add(TestUtil.randomUnicodeString(random())); |
| } |
| |
| String softDeletesField = |
| random().nextBoolean() ? TestUtil.randomUnicodeString(random()) : null; |
| FieldInfos.Builder builder = |
| new FieldInfos.Builder(new FieldInfos.FieldNumbers(softDeletesField)); |
| |
| for (String field : fieldNames) { |
| IndexableFieldType fieldType = randomFieldType(random()); |
| FieldInfo fi = builder.getOrAdd(field); |
| IndexOptions indexOptions = fieldType.indexOptions(); |
| if (indexOptions != IndexOptions.NONE) { |
| fi.setIndexOptions(indexOptions); |
| if (fieldType.omitNorms()) { |
| fi.setOmitsNorms(); |
| } |
| } |
| fi.setDocValuesType(fieldType.docValuesType()); |
| if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) { |
| if (random().nextBoolean()) { |
| fi.setStorePayloads(); |
| } |
| } |
| |
| if (fieldType.pointDimensionCount() > 0) { |
| fi.setPointDimensions( |
| fieldType.pointDimensionCount(), |
| fieldType.pointIndexDimensionCount(), |
| fieldType.pointNumBytes()); |
| } |
| |
| addAttributes(fi); |
| } |
| FieldInfos infos = builder.finish(); |
| codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT); |
| FieldInfos infos2 = codec.fieldInfosFormat().read(dir, segmentInfo, "", IOContext.DEFAULT); |
| assertEquals(infos, infos2); |
| dir.close(); |
| } |
| |
| private final IndexableFieldType randomFieldType(Random r) { |
| FieldType type = new FieldType(); |
| |
| if (r.nextBoolean()) { |
| IndexOptions[] values = IndexOptions.values(); |
| type.setIndexOptions(values[r.nextInt(values.length)]); |
| type.setOmitNorms(r.nextBoolean()); |
| |
| if (r.nextBoolean()) { |
| type.setStoreTermVectors(true); |
| if (type.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) { |
| type.setStoreTermVectorPositions(r.nextBoolean()); |
| type.setStoreTermVectorOffsets(r.nextBoolean()); |
| if (type.storeTermVectorPositions()) { |
| type.setStoreTermVectorPayloads(r.nextBoolean()); |
| } |
| } |
| } |
| } |
| |
| if (r.nextBoolean()) { |
| DocValuesType[] values = DocValuesType.values(); |
| type.setDocValuesType(values[r.nextInt(values.length)]); |
| } |
| |
| if (r.nextBoolean()) { |
| int dimension = 1 + r.nextInt(PointValues.MAX_DIMENSIONS); |
| int indexDimension = 1 + r.nextInt(Math.min(dimension, PointValues.MAX_INDEX_DIMENSIONS)); |
| int dimensionNumBytes = 1 + r.nextInt(PointValues.MAX_NUM_BYTES); |
| type.setDimensions(dimension, indexDimension, dimensionNumBytes); |
| } |
| |
| return type; |
| } |
| |
| /** Hook to add any codec attributes to fieldinfo instances added in this test. */ |
| protected void addAttributes(FieldInfo fi) {} |
| |
| /** equality for entirety of fieldinfos */ |
| protected void assertEquals(FieldInfos expected, FieldInfos actual) { |
| assertEquals(expected.size(), actual.size()); |
| for (FieldInfo expectedField : expected) { |
| FieldInfo actualField = actual.fieldInfo(expectedField.number); |
| assertNotNull(actualField); |
| assertEquals(expectedField, actualField); |
| } |
| } |
| |
| /** equality for two individual fieldinfo objects */ |
| protected void assertEquals(FieldInfo expected, FieldInfo actual) { |
| assertEquals(expected.number, actual.number); |
| assertEquals(expected.name, actual.name); |
| assertEquals(expected.getDocValuesType(), actual.getDocValuesType()); |
| assertEquals(expected.getIndexOptions(), actual.getIndexOptions()); |
| assertEquals(expected.hasNorms(), actual.hasNorms()); |
| assertEquals(expected.hasPayloads(), actual.hasPayloads()); |
| assertEquals(expected.hasVectors(), actual.hasVectors()); |
| assertEquals(expected.omitsNorms(), actual.omitsNorms()); |
| assertEquals(expected.getDocValuesGen(), actual.getDocValuesGen()); |
| } |
| |
| /** Returns a new fake segment */ |
| protected static SegmentInfo newSegmentInfo(Directory dir, String name) { |
| Version minVersion = random().nextBoolean() ? null : Version.LATEST; |
| return new SegmentInfo(dir, Version.LATEST, minVersion, name, 10000, false, Codec.getDefault(), Collections.emptyMap(), StringHelper.randomId(), Collections.emptyMap(), null); |
| } |
| |
| @Override |
| protected void addRandomFields(Document doc) { |
| doc.add(new StoredField("foobar", TestUtil.randomSimpleString(random()))); |
| } |
| |
| @Override |
| public void testRamBytesUsed() throws IOException { |
| assumeTrue("not applicable for this format", true); |
| } |
| } |