| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.solr.uninverting; |
| |
| import java.io.IOException; |
| |
| import org.apache.lucene.document.NumericDocValuesField; |
| import org.apache.lucene.index.BinaryDocValues; |
| import org.apache.lucene.index.IndexReader; // javadocs |
| import org.apache.lucene.index.LeafReader; |
| import org.apache.lucene.index.NumericDocValues; |
| import org.apache.lucene.index.SortedDocValues; |
| import org.apache.lucene.index.SortedSetDocValues; |
| import org.apache.lucene.index.Terms; |
| import org.apache.lucene.index.TermsEnum; |
| import org.apache.solr.legacy.LegacyNumericUtils; |
| import org.apache.lucene.util.Accountable; |
| import org.apache.lucene.util.Bits; |
| import org.apache.lucene.util.BytesRef; |
| import org.apache.lucene.util.NumericUtils; |
| import org.apache.lucene.util.RamUsageEstimator; |
| |
| /** |
| * Expert: Maintains caches of term values. |
| * |
| * <p>Created: May 19, 2004 11:13:14 AM |
| * |
| * @since lucene 1.4 |
| * |
| * @lucene.internal |
| */ |
| public interface FieldCache { |
| |
| /** |
| * Placeholder indicating creation of this cache is currently in-progress. |
| */ |
| public static final class CreationPlaceholder implements Accountable { |
| Accountable value; |
| |
| @Override |
| public long ramBytesUsed() { |
| // don't call on the in-progress value, might make things angry. |
| return RamUsageEstimator.NUM_BYTES_OBJECT_REF; |
| } |
| } |
| |
| /** |
| * interface to all parsers. It is used to parse different numeric types. |
| */ |
| public interface Parser { |
| |
| /** |
| * Pulls a {@link TermsEnum} from the given {@link Terms}. This method allows certain parsers |
| * to filter the actual TermsEnum before the field cache is filled. |
| * |
| * @param terms the {@link Terms} instance to create the {@link TermsEnum} from. |
| * @return a possibly filtered {@link TermsEnum} instance, this method must not return <code>null</code>. |
| * @throws IOException if an {@link IOException} occurs |
| * @deprecated index with Points instead |
| */ |
| @Deprecated |
| public TermsEnum termsEnum(Terms terms) throws IOException; |
| |
| /** Parse's this field's value */ |
| public long parseValue(BytesRef term); |
| } |
| |
| /** |
| * Base class for points parsers. These parsers do not use the inverted index, but instead |
| * uninvert point data. |
| * |
| * This abstraction can be cleaned up when Parser.termsEnum is removed. |
| */ |
| public abstract class PointParser implements Parser { |
| public final TermsEnum termsEnum(Terms terms) throws IOException { |
| throw new UnsupportedOperationException("makes no sense for parsing points"); |
| } |
| } |
| |
| /** Expert: The cache used internally by sorting and range query classes. */ |
| public static FieldCache DEFAULT = new FieldCacheImpl(); |
| |
| /** |
| * A parser instance for int values encoded by {@link org.apache.lucene.util.NumericUtils}, e.g. when indexed |
| * via {@link org.apache.lucene.document.IntPoint}. |
| */ |
| public static final Parser INT_POINT_PARSER = new PointParser() { |
| @Override |
| public long parseValue(BytesRef point) { |
| return NumericUtils.sortableBytesToInt(point.bytes, point.offset); |
| } |
| |
| @Override |
| public String toString() { |
| return FieldCache.class.getName()+".INT_POINT_PARSER"; |
| } |
| }; |
| |
| /** |
| * A parser instance for long values encoded by {@link org.apache.lucene.util.NumericUtils}, e.g. when indexed |
| * via {@link org.apache.lucene.document.LongPoint}. |
| */ |
| public static final Parser LONG_POINT_PARSER = new PointParser() { |
| @Override |
| public long parseValue(BytesRef point) { |
| return NumericUtils.sortableBytesToLong(point.bytes, point.offset); |
| } |
| |
| @Override |
| public String toString() { |
| return FieldCache.class.getName()+".LONG_POINT_PARSER"; |
| } |
| }; |
| |
| /** |
| * A parser instance for float values encoded by {@link org.apache.lucene.util.NumericUtils}, e.g. when indexed |
| * via {@link org.apache.lucene.document.FloatPoint}. |
| */ |
| public static final Parser FLOAT_POINT_PARSER = new PointParser() { |
| @Override |
| public long parseValue(BytesRef point) { |
| return NumericUtils.sortableFloatBits(NumericUtils.sortableBytesToInt(point.bytes, point.offset)); |
| } |
| |
| @Override |
| public String toString() { |
| return FieldCache.class.getName()+".FLOAT_POINT_PARSER"; |
| } |
| }; |
| |
| /** |
| * A parser instance for double values encoded by {@link org.apache.lucene.util.NumericUtils}, e.g. when indexed |
| * via {@link org.apache.lucene.document.DoublePoint}. |
| */ |
| public static final Parser DOUBLE_POINT_PARSER = new PointParser() { |
| @Override |
| public long parseValue(BytesRef point) { |
| return NumericUtils.sortableDoubleBits(NumericUtils.sortableBytesToLong(point.bytes, point.offset)); |
| } |
| |
| @Override |
| public String toString() { |
| return FieldCache.class.getName()+".DOUBLE_POINT_PARSER"; |
| } |
| }; |
| |
| /** |
| * A parser instance for int values encoded by {@link org.apache.solr.legacy.LegacyNumericUtils}, e.g. when indexed |
| * via {@link org.apache.solr.legacy.LegacyIntField}/{@link org.apache.solr.legacy.LegacyNumericTokenStream}. |
| * @deprecated Index with points and use {@link #INT_POINT_PARSER} instead. |
| */ |
| @Deprecated |
| public static final Parser LEGACY_INT_PARSER = new Parser() { |
| @Override |
| public long parseValue(BytesRef term) { |
| return LegacyNumericUtils.prefixCodedToInt(term); |
| } |
| |
| @Override |
| public TermsEnum termsEnum(Terms terms) throws IOException { |
| return LegacyNumericUtils.filterPrefixCodedInts(terms.iterator()); |
| } |
| |
| @Override |
| public String toString() { |
| return FieldCache.class.getName()+".LEGACY_INT_PARSER"; |
| } |
| }; |
| |
| /** |
| * A parser instance for float values encoded with {@link org.apache.solr.legacy.LegacyNumericUtils}, e.g. when indexed |
| * via {@link org.apache.solr.legacy.LegacyFloatField}/{@link org.apache.solr.legacy.LegacyNumericTokenStream}. |
| * @deprecated Index with points and use {@link #FLOAT_POINT_PARSER} instead. |
| */ |
| @Deprecated |
| public static final Parser LEGACY_FLOAT_PARSER = new Parser() { |
| @Override |
| public long parseValue(BytesRef term) { |
| int val = LegacyNumericUtils.prefixCodedToInt(term); |
| if (val<0) val ^= 0x7fffffff; |
| return val; |
| } |
| |
| @Override |
| public String toString() { |
| return FieldCache.class.getName()+".LEGACY_FLOAT_PARSER"; |
| } |
| |
| @Override |
| public TermsEnum termsEnum(Terms terms) throws IOException { |
| return LegacyNumericUtils.filterPrefixCodedInts(terms.iterator()); |
| } |
| }; |
| |
| /** |
| * A parser instance for long values encoded by {@link org.apache.solr.legacy.LegacyNumericUtils}, e.g. when indexed |
| * via {@link org.apache.solr.legacy.LegacyLongField}/{@link org.apache.solr.legacy.LegacyNumericTokenStream}. |
| * @deprecated Index with points and use {@link #LONG_POINT_PARSER} instead. |
| */ |
| @Deprecated |
| public static final Parser LEGACY_LONG_PARSER = new Parser() { |
| @Override |
| public long parseValue(BytesRef term) { |
| return LegacyNumericUtils.prefixCodedToLong(term); |
| } |
| @Override |
| public String toString() { |
| return FieldCache.class.getName()+".LEGACY_LONG_PARSER"; |
| } |
| |
| @Override |
| public TermsEnum termsEnum(Terms terms) throws IOException { |
| return LegacyNumericUtils.filterPrefixCodedLongs(terms.iterator()); |
| } |
| }; |
| |
| /** |
| * A parser instance for double values encoded with {@link org.apache.solr.legacy.LegacyNumericUtils}, e.g. when indexed |
| * via {@link org.apache.solr.legacy.LegacyDoubleField}/{@link org.apache.solr.legacy.LegacyNumericTokenStream}. |
| * @deprecated Index with points and use {@link #DOUBLE_POINT_PARSER} instead. |
| */ |
| @Deprecated |
| public static final Parser LEGACY_DOUBLE_PARSER = new Parser() { |
| @Override |
| public long parseValue(BytesRef term) { |
| long val = LegacyNumericUtils.prefixCodedToLong(term); |
| if (val<0) val ^= 0x7fffffffffffffffL; |
| return val; |
| } |
| @Override |
| public String toString() { |
| return FieldCache.class.getName()+".LEGACY_DOUBLE_PARSER"; |
| } |
| |
| @Override |
| public TermsEnum termsEnum(Terms terms) throws IOException { |
| return LegacyNumericUtils.filterPrefixCodedLongs(terms.iterator()); |
| } |
| }; |
| |
| /** Checks the internal cache for an appropriate entry, and if none is found, |
| * reads the terms/points in <code>field</code> and returns a bit set at the size of |
| * <code>reader.maxDoc()</code>, with turned on bits for each docid that |
| * does have a value for this field. |
| * @param parser May be {@code null} if coming from the inverted index, otherwise |
| * can be a {@link PointParser} to compute from point values. |
| */ |
| public Bits getDocsWithField(LeafReader reader, String field, Parser parser) throws IOException; |
| |
| /** |
| * Returns a {@link NumericDocValues} over the values found in documents in the given |
| * field. If the field was indexed as {@link NumericDocValuesField}, it simply |
| * uses {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)} to read the values. |
| * Otherwise, it checks the internal cache for an appropriate entry, and if |
| * none is found, reads the terms/points in <code>field</code> as longs and returns |
| * an array of size <code>reader.maxDoc()</code> of the value each document |
| * has in the given field. |
| * |
| * @param reader |
| * Used to get field values. |
| * @param field |
| * Which field contains the longs. |
| * @param parser |
| * Computes long for string values. May be {@code null} if the |
| * requested field was indexed as {@link NumericDocValuesField} or |
| * {@link org.apache.solr.legacy.LegacyLongField}. |
| * @return The values in the given field for each document. |
| * @throws IOException |
| * If any error occurs. |
| */ |
| public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser) throws IOException; |
| |
| /** Checks the internal cache for an appropriate entry, and if none |
| * is found, reads the term values in <code>field</code> |
| * and returns a {@link BinaryDocValues} instance, providing a |
| * method to retrieve the term (as a BytesRef) per document. |
| * @param reader Used to get field values. |
| * @param field Which field contains the strings. |
| * @return The values in the given field for each document. |
| * @throws IOException If any error occurs. |
| */ |
| public BinaryDocValues getTerms(LeafReader reader, String field) throws IOException; |
| |
| /** Expert: just like {@link #getTerms(org.apache.lucene.index.LeafReader,String)}, |
| * but you can specify whether more RAM should be consumed in exchange for |
| * faster lookups (default is "true"). Note that the |
| * first call for a given reader and field "wins", |
| * subsequent calls will share the same cache entry. */ |
| public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException; |
| |
| /** Checks the internal cache for an appropriate entry, and if none |
| * is found, reads the term values in <code>field</code> |
| * and returns a {@link SortedDocValues} instance, |
| * providing methods to retrieve sort ordinals and terms |
| * (as a ByteRef) per document. |
| * @param reader Used to get field values. |
| * @param field Which field contains the strings. |
| * @return The values in the given field for each document. |
| * @throws IOException If any error occurs. |
| */ |
| public SortedDocValues getTermsIndex(LeafReader reader, String field) throws IOException; |
| |
| /** Expert: just like {@link |
| * #getTermsIndex(org.apache.lucene.index.LeafReader,String)}, but you can specify |
| * whether more RAM should be consumed in exchange for |
| * faster lookups (default is "true"). Note that the |
| * first call for a given reader and field "wins", |
| * subsequent calls will share the same cache entry. */ |
| public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException; |
| |
| /** Can be passed to {@link #getDocTermOrds} to filter for 32-bit numeric terms */ |
| public static final BytesRef INT32_TERM_PREFIX = new BytesRef(new byte[] { LegacyNumericUtils.SHIFT_START_INT }); |
| /** Can be passed to {@link #getDocTermOrds} to filter for 64-bit numeric terms */ |
| public static final BytesRef INT64_TERM_PREFIX = new BytesRef(new byte[] { LegacyNumericUtils.SHIFT_START_LONG }); |
| |
| /** |
| * Checks the internal cache for an appropriate entry, and if none is found, reads the term values |
| * in <code>field</code> and returns a {@link DocTermOrds} instance, providing a method to retrieve |
| * the terms (as ords) per document. |
| * |
| * @param reader Used to build a {@link DocTermOrds} instance |
| * @param field Which field contains the strings. |
| * @param prefix prefix for a subset of the terms which should be uninverted. Can be null or |
| * {@link #INT32_TERM_PREFIX} or {@link #INT64_TERM_PREFIX} |
| * |
| * @return a {@link DocTermOrds} instance |
| * @throws IOException If any error occurs. |
| */ |
| public SortedSetDocValues getDocTermOrds(LeafReader reader, String field, BytesRef prefix) throws IOException; |
| |
| /** |
| * EXPERT: A unique Identifier/Description for each item in the FieldCache. |
| * Can be useful for logging/debugging. |
| * @lucene.experimental |
| */ |
| public final class CacheEntry { |
| |
| private final Object readerKey; |
| private final String fieldName; |
| private final Class<?> cacheType; |
| private final Object custom; |
| private final Accountable value; |
| |
| public CacheEntry(IndexReader.CacheKey readerKey, String fieldName, |
| Class<?> cacheType, |
| Object custom, |
| Accountable value) { |
| this.readerKey = readerKey; |
| this.fieldName = fieldName; |
| this.cacheType = cacheType; |
| this.custom = custom; |
| this.value = value; |
| } |
| |
| public Object getReaderKey() { |
| return readerKey; |
| } |
| |
| public String getFieldName() { |
| return fieldName; |
| } |
| |
| public Class<?> getCacheType() { |
| return cacheType; |
| } |
| |
| public Object getCustom() { |
| return custom; |
| } |
| |
| public Accountable getValue() { |
| return value; |
| } |
| |
| /** |
| * The most recently estimated size of the value, null unless |
| * estimateSize has been called. |
| */ |
| public String getEstimatedSize() { |
| long bytesUsed = value == null ? 0L : value.ramBytesUsed(); |
| return RamUsageEstimator.humanReadableUnits(bytesUsed); |
| } |
| |
| @Override |
| public String toString() { |
| StringBuilder b = new StringBuilder(100); |
| b.append("segment='").append(getReaderKey().toString()).append("', "); |
| b.append("field='").append(getFieldName()).append("', "); |
| String s = getEstimatedSize(); |
| b.append("size =~ ").append(s); |
| |
| return b.toString(); |
| } |
| } |
| |
| /** |
| * EXPERT: Generates an array of CacheEntry objects representing all items |
| * currently in the FieldCache. |
| * <p> |
| * NOTE: These CacheEntry objects maintain a strong reference to the |
| * Cached Values. Maintaining references to a CacheEntry the AtomicIndexReader |
| * associated with it has garbage collected will prevent the Value itself |
| * from being garbage collected when the Cache drops the WeakReference. |
| * </p> |
| * @lucene.experimental |
| */ |
| public CacheEntry[] getCacheEntries(); |
| |
| /** |
| * <p> |
| * EXPERT: Instructs the FieldCache to forcibly expunge all entries |
| * from the underlying caches. This is intended only to be used for |
| * test methods as a way to ensure a known base state of the Cache |
| * (with out needing to rely on GC to free WeakReferences). |
| * It should not be relied on for "Cache maintenance" in general |
| * application code. |
| * </p> |
| * @lucene.experimental |
| */ |
| public void purgeAllCaches(); |
| |
| /** |
| * Expert: drops all cache entries associated with this |
| * reader {@link org.apache.lucene.index.IndexReader.CacheHelper#getKey()}. |
| * NOTE: this cache key must |
| * precisely match the reader that the cache entry is |
| * keyed on. If you pass a top-level reader, it usually |
| * will have no effect as Lucene now caches at the segment |
| * reader level. |
| */ |
| public void purgeByCacheKey(IndexReader.CacheKey coreCacheKey); |
| |
| } |