| Index: src/java/org/apache/lucene/document/Field.java
|
| ===================================================================
|
| --- src/java/org/apache/lucene/document/Field.java (revision 763566)
|
| +++ src/java/org/apache/lucene/document/Field.java (working copy)
|
| @@ -329,6 +329,8 @@
|
| if (index == Index.NO) { |
| this.isIndexed = false; |
| this.isTokenized = false; |
| + this.omitTermFreqAndPositions = false; |
| + this.omitNorms = true; |
| } else if (index == Index.ANALYZED) { |
| this.isIndexed = true; |
| this.isTokenized = true; |
| @@ -492,6 +494,8 @@
|
| |
| isIndexed = false; |
| isTokenized = false; |
| + omitTermFreqAndPositions = false; |
| + omitNorms = true; |
| |
| isBinary = true; |
| binaryLength = length; |
| Index: src/java/org/apache/lucene/index/FieldInfo.java
|
| ===================================================================
|
| --- src/java/org/apache/lucene/index/FieldInfo.java (revision 763566)
|
| +++ src/java/org/apache/lucene/index/FieldInfo.java (working copy)
|
| @@ -38,12 +38,21 @@
|
| name = na; |
| isIndexed = tk; |
| number = nu; |
| - this.storeTermVector = storeTermVector; |
| - this.storeOffsetWithTermVector = storeOffsetWithTermVector; |
| - this.storePositionWithTermVector = storePositionWithTermVector; |
| - this.omitNorms = omitNorms; |
| - this.storePayloads = storePayloads; |
| - this.omitTermFreqAndPositions = omitTermFreqAndPositions; |
| + if (isIndexed) { |
| + this.storeTermVector = storeTermVector; |
| + this.storeOffsetWithTermVector = storeOffsetWithTermVector; |
| + this.storePositionWithTermVector = storePositionWithTermVector; |
| + this.storePayloads = storePayloads; |
| + this.omitNorms = omitNorms; |
| + this.omitTermFreqAndPositions = omitTermFreqAndPositions; |
| + } else { // for non-indexed fields, leave defaults |
| + this.storeTermVector = false; |
| + this.storeOffsetWithTermVector = false; |
| + this.storePositionWithTermVector = false; |
| + this.storePayloads = false; |
| + this.omitNorms = true; |
| + this.omitTermFreqAndPositions = false; |
| + } |
| } |
| |
| public Object clone() { |
| @@ -56,23 +65,25 @@
|
| if (this.isIndexed != isIndexed) { |
| this.isIndexed = true; // once indexed, always index |
| } |
| - if (this.storeTermVector != storeTermVector) { |
| - this.storeTermVector = true; // once vector, always vector |
| + if (isIndexed) { // if updated field data is not for indexing, leave the updates out |
| + if (this.storeTermVector != storeTermVector) { |
| + this.storeTermVector = true; // once vector, always vector |
| + } |
| + if (this.storePositionWithTermVector != storePositionWithTermVector) { |
| + this.storePositionWithTermVector = true; // once vector, always vector |
| + } |
| + if (this.storeOffsetWithTermVector != storeOffsetWithTermVector) { |
| + this.storeOffsetWithTermVector = true; // once vector, always vector |
| + } |
| + if (this.storePayloads != storePayloads) { |
| + this.storePayloads = true; |
| + } |
| + if (this.omitNorms != omitNorms) { |
| + this.omitNorms = false; // once norms are stored, always store |
| + } |
| + if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) { |
| + this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life |
| + } |
| } |
| - if (this.storePositionWithTermVector != storePositionWithTermVector) { |
| - this.storePositionWithTermVector = true; // once vector, always vector |
| - } |
| - if (this.storeOffsetWithTermVector != storeOffsetWithTermVector) { |
| - this.storeOffsetWithTermVector = true; // once vector, always vector |
| - } |
| - if (this.omitNorms != omitNorms) { |
| - this.omitNorms = false; // once norms are stored, always store |
| - } |
| - if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) { |
| - this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life |
| - } |
| - if (this.storePayloads != storePayloads) { |
| - this.storePayloads = true; |
| - } |
| } |
| } |
| Index: src/java/org/apache/lucene/index/FieldInfos.java
|
| ===================================================================
|
| --- src/java/org/apache/lucene/index/FieldInfos.java (revision 763566)
|
| +++ src/java/org/apache/lucene/index/FieldInfos.java (working copy)
|
| @@ -84,7 +84,7 @@
|
| while (fieldIterator.hasNext()) { |
| Fieldable field = (Fieldable) fieldIterator.next(); |
| add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), |
| - field.isStoreOffsetWithTermVector(), field.getOmitNorms()); |
| + field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTf()); |
| } |
| } |
| |
| Index: src/java/org/apache/lucene/index/FieldsReader.java
|
| ===================================================================
|
| --- src/java/org/apache/lucene/index/FieldsReader.java (revision 763566)
|
| +++ src/java/org/apache/lucene/index/FieldsReader.java (working copy)
|
| @@ -322,6 +322,7 @@
|
| //skip over the part that we aren't loading |
| fieldsStream.seek(pointer + toRead); |
| f.setOmitNorms(fi.omitNorms); |
| + f.setOmitTf(fi.omitTermFreqAndPositions); |
| } else { |
| int length = fieldsStream.readVInt(); |
| long pointer = fieldsStream.getFilePointer(); |
| @@ -332,6 +333,7 @@
|
| fieldsStream.skipChars(length); |
| f = new LazyField(fi.name, store, index, termVector, length, pointer, binary); |
| f.setOmitNorms(fi.omitNorms); |
| + f.setOmitTf(fi.omitTermFreqAndPositions); |
| } |
| doc.add(f); |
| } |
| @@ -365,7 +367,6 @@
|
| doc.add(new Field(fi.name, uncompress(b), Field.Store.COMPRESS)); |
| else |
| doc.add(new Field(fi.name, b, Field.Store.YES)); |
| - |
| } else { |
| Field.Store store = Field.Store.YES; |
| Field.Index index = getIndexType(fi, tokenize); |
| @@ -383,6 +384,7 @@
|
| store, |
| index, |
| termVector); |
| + f.setOmitTf(fi.omitTermFreqAndPositions); |
| f.setOmitNorms(fi.omitNorms); |
| } else { |
| f = new Field(fi.name, // name |
| @@ -390,6 +392,7 @@
|
| store, |
| index, |
| termVector); |
| + f.setOmitTf(fi.omitTermFreqAndPositions); |
| f.setOmitNorms(fi.omitNorms); |
| } |
| doc.add(f); |
| @@ -641,6 +644,7 @@
|
| this.name = fi.name.intern(); |
| this.isIndexed = fi.isIndexed; |
| this.omitNorms = fi.omitNorms; |
| + this.omitTermFreqAndPositions = fi.omitTermFreqAndPositions; |
| this.storeOffsetWithTermVector = fi.storeOffsetWithTermVector; |
| this.storePositionWithTermVector = fi.storePositionWithTermVector; |
| this.storeTermVector = fi.storeTermVector; |
| Index: src/test/org/apache/lucene/index/DocHelper.java
|
| ===================================================================
|
| --- src/test/org/apache/lucene/index/DocHelper.java (revision 763566)
|
| +++ src/test/org/apache/lucene/index/DocHelper.java (working copy)
|
| @@ -64,6 +64,14 @@
|
| public static Field noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT, |
| Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); |
| |
| + public static final String NO_TF_TEXT = "analyzed with no tf and positions"; |
| + public static final String NO_TF_KEY = "omitTermFreqAndPositions"; |
| + public static Field noTFField = new Field(NO_TF_KEY, NO_TF_TEXT, |
| + Field.Store.YES, Field.Index.ANALYZED); |
| + static { |
| + noTFField.setOmitTermFreqAndPositions(true); |
| + } |
| + |
| public static final String UNINDEXED_FIELD_TEXT = "unindexed field text"; |
| public static final String UNINDEXED_FIELD_KEY = "unIndField"; |
| public static Field unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT, |
| @@ -119,6 +127,7 @@
|
| compressedTextField2, |
| keyField, |
| noNormsField, |
| + noTFField, |
| unIndField, |
| unStoredField1, |
| unStoredField2, |
| @@ -139,6 +148,7 @@
|
| public static Map notermvector=new HashMap(); |
| public static Map lazy= new HashMap(); |
| public static Map noNorms=new HashMap(); |
| + public static Map noTf=new HashMap(); |
| |
| static { |
| //Initialize the large Lazy Field |
| @@ -167,6 +177,7 @@
|
| if (f.isStored()) add(stored,f); |
| else add(unstored,f); |
| if (f.getOmitNorms()) add(noNorms,f); |
| + if (f.getOmitTf()) add(noTf,f); |
| if (f.isLazy()) add(lazy, f); |
| } |
| } |
| @@ -186,6 +197,7 @@
|
| nameValues.put(TEXT_FIELD_3_KEY, FIELD_3_TEXT); |
| nameValues.put(KEYWORD_FIELD_KEY, KEYWORD_TEXT); |
| nameValues.put(NO_NORMS_KEY, NO_NORMS_TEXT); |
| + nameValues.put(NO_TF_KEY, NO_TF_TEXT); |
| nameValues.put(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT); |
| nameValues.put(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT); |
| nameValues.put(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT); |
| Index: src/test/org/apache/lucene/index/TestDocumentWriter.java
|
| ===================================================================
|
| --- src/test/org/apache/lucene/index/TestDocumentWriter.java (revision 763566)
|
| +++ src/test/org/apache/lucene/index/TestDocumentWriter.java (working copy)
|
| @@ -259,14 +259,13 @@
|
| doc.add(new Field("f2", "v1", Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS)); |
| doc.add(new Field("f2", "v2", Store.YES, Index.NOT_ANALYZED, TermVector.NO)); |
| |
| - RAMDirectory ram = new RAMDirectory(); |
| - IndexWriter writer = new IndexWriter(ram, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); |
| + IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); |
| writer.addDocument(doc); |
| writer.close(); |
| |
| - _TestUtil.checkIndex(ram); |
| + _TestUtil.checkIndex(dir); |
| |
| - IndexReader reader = IndexReader.open(ram); |
| + IndexReader reader = IndexReader.open(dir); |
| // f1 |
| TermFreqVector tfv1 = reader.getTermFreqVector(0, "f1"); |
| assertNotNull(tfv1); |
| @@ -276,4 +275,37 @@
|
| assertNotNull(tfv2); |
| assertEquals("the 'with_tv' setting should rule!",2,tfv2.getTerms().length); |
| } |
| + |
| + /** |
| + * Test adding two fields with the same name, one indexed |
| + * the other stored only. The omitNorms and omitTermFreqAndPositions setting |
| + * of the stored field should not affect the indexed one (LUCENE-1590) |
| + */ |
| + public void testLUCENE_1590() throws Exception { |
| + Document doc = new Document(); |
| + // f1 has no norms |
| + doc.add(new Field("f1", "v1", Store.NO, Index.ANALYZED_NO_NORMS)); |
| + doc.add(new Field("f1", "v2", Store.YES, Index.NO)); |
| + // f2 has no TF |
| + Field f = new Field("f2", "v1", Store.NO, Index.ANALYZED); |
| + f.setOmitTermFreqAndPositions(true); |
| + doc.add(f); |
| + doc.add(new Field("f2", "v2", Store.YES, Index.NO)); |
| + |
| + IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); |
| + writer.addDocument(doc); |
| + writer.optimize(); // be sure to have a single segment |
| + writer.close(); |
| + |
| + _TestUtil.checkIndex(dir); |
| + |
| + SegmentReader reader = (SegmentReader) IndexReader.open(dir); |
| + FieldInfos fi = reader.fieldInfos(); |
| + // f1 |
| + assertFalse("f1 should have no norms", reader.hasNorms("f1")); |
| + assertFalse("omitTermFreqAndPositions field bit should not be set for f1", fi.fieldInfo("f1").omitTermFreqAndPositions); |
| + // f2 |
| + assertTrue("f2 should have norms", reader.hasNorms("f2")); |
| + assertTrue("omitTermFreqAndPositions field bit should be set for f2", fi.fieldInfo("f2").omitTermFreqAndPositions); |
| + } |
| } |
| Index: src/test/org/apache/lucene/index/TestFieldsReader.java
|
| ===================================================================
|
| --- src/test/org/apache/lucene/index/TestFieldsReader.java (revision 763566)
|
| +++ src/test/org/apache/lucene/index/TestFieldsReader.java (working copy)
|
| @@ -72,6 +72,7 @@
|
| assertTrue(field.isStoreOffsetWithTermVector() == true); |
| assertTrue(field.isStorePositionWithTermVector() == true); |
| assertTrue(field.getOmitNorms() == false); |
| + assertTrue(field.getOmitTf() == false); |
| |
| field = doc.getField(DocHelper.TEXT_FIELD_3_KEY); |
| assertTrue(field != null); |
| @@ -79,8 +80,15 @@
|
| assertTrue(field.isStoreOffsetWithTermVector() == false); |
| assertTrue(field.isStorePositionWithTermVector() == false); |
| assertTrue(field.getOmitNorms() == true); |
| + assertTrue(field.getOmitTf() == false); |
| |
| - |
| + field = doc.getField(DocHelper.NO_TF_KEY); |
| + assertTrue(field != null); |
| + assertTrue(field.isTermVectorStored() == false); |
| + assertTrue(field.isStoreOffsetWithTermVector() == false); |
| + assertTrue(field.isStorePositionWithTermVector() == false); |
| + assertTrue(field.getOmitNorms() == false); |
| + assertTrue(field.getOmitTf() == true); |
| reader.close(); |
| } |
| |
