lucene/misc/src/java/org/apache/lucene/search/DocValuesStats.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.lucene.search;

 import java.io.IOException;

 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;

 /** Holds statistics for a DocValues field. */
 public abstract class DocValuesStats<T> {

   private int missing = 0;
   private int count = 0;

   protected final String field;

   protected T min;
   protected T max;

   protected DocValuesStats(String field, T initialMin, T initialMax) {
     this.field = field;
     this.min = initialMin;
     this.max = initialMax;
   }

   /**
    * Called after {@link #accumulate(int)} was processed and verified that the document has a value for
    * the field. Implementations should update the statistics based on the value of the current document.
    *
    * @param count
    *          the updated number of documents with value for this field.
    */
   protected abstract void doAccumulate(int count) throws IOException;

   /**
    * Initializes this object with the given reader context. Returns whether stats can be computed for this segment (i.e.
    * it does have the requested DocValues field).
    */
   protected abstract boolean init(LeafReaderContext context) throws IOException;

   /** Returns whether the given document has a value for the requested DocValues field. */
   protected abstract boolean hasValue(int doc) throws IOException;

   final void accumulate(int doc) throws IOException {
     if (hasValue(doc)) {
       ++count;
       doAccumulate(count);
     } else {
       ++missing;
     }
   }

   final void addMissing() {
     ++missing;
   }

   /** The field for which these stats were computed. */
   public final String field() {
     return field;
   }

   /** The number of documents which have a value of the field. */
   public final int count() {
     return count;
   }

   /** The number of documents which do not have a value of the field. */
   public final int missing() {
     return missing;
   }

   /** The minimum value of the field. Undefined when {@link #count()} is zero. */
   public final T min() {
     return min;
   }

   /** The maximum value of the field. Undefined when {@link #count()} is zero. */
   public final T max() {
     return max;
   }

   /** Holds statistics for a numeric DocValues field. */
   public static abstract class NumericDocValuesStats<T extends Number> extends DocValuesStats<T> {

     protected double mean = 0.0;
     protected double variance = 0.0;

     protected NumericDocValues ndv;

     protected NumericDocValuesStats(String field, T initialMin, T initialMax) {
       super(field, initialMin, initialMax);
     }

     @Override
     protected final boolean init(LeafReaderContext context) throws IOException {
       ndv = context.reader().getNumericDocValues(field);
       return ndv != null;
     }

     @Override
     protected final boolean hasValue(int doc) throws IOException {
       return ndv.advanceExact(doc);
     }

     /** The mean of all values of the field. */
     public final double mean() {
       return mean;
     }

     /** Returns the variance of all values of the field. */
     public final double variance() {
       int count = count();
       return count > 0 ? variance / count : 0;
     }

     /** Returns the stdev of all values of the field. */
     public final double stdev() {
       return Math.sqrt(variance());
     }

     /** Returns the sum of values of the field. Note that if the values are large, the {@code sum} might overflow. */
     public abstract T sum();
   }

   /** Holds DocValues statistics for a numeric field storing {@code long} values. */
   public static final class LongDocValuesStats extends NumericDocValuesStats<Long> {

     // To avoid boxing 'long' to 'Long' while the sum is computed, declare it as private variable.
     private long sum = 0;

     public LongDocValuesStats(String field) {
       super(field, Long.MAX_VALUE, Long.MIN_VALUE);
     }

     @Override
     protected void doAccumulate(int count) throws IOException {
       long val = ndv.longValue();
       if (val > max) {
         max = val;
       }
       if (val < min) {
         min = val;
       }
       sum += val;
       double oldMean = mean;
       mean += (val - mean) / count;
       variance += (val - mean) * (val - oldMean);
     }

     @Override
     public Long sum() {
       return sum;
     }
   }

   /** Holds DocValues statistics for a numeric field storing {@code double} values. */
   public static final class DoubleDocValuesStats extends NumericDocValuesStats<Double> {

     // To avoid boxing 'double' to 'Double' while the sum is computed, declare it as private variable.
     private double sum = 0;

     public DoubleDocValuesStats(String field) {
       super(field, Double.MAX_VALUE, Double.MIN_VALUE);
     }

     @Override
     protected void doAccumulate(int count) throws IOException {
       double val = Double.longBitsToDouble(ndv.longValue());
       if (Double.compare(val, max) > 0) {
         max = val;
       }
       if (Double.compare(val, min) < 0) {
         min = val;
       }
       sum += val;
       double oldMean = mean;
       mean += (val - mean) / count;
       variance += (val - mean) * (val - oldMean);
     }

     @Override
     public Double sum() {
       return sum;
     }
   }

   /** Holds statistics for a sorted-numeric DocValues field. */
   public static abstract class SortedNumericDocValuesStats<T extends Number> extends DocValuesStats<T> {

     protected long valuesCount = 0;
     protected double mean = 0.0;
     protected double variance = 0.0;

     protected SortedNumericDocValues sndv;

     protected SortedNumericDocValuesStats(String field, T initialMin, T initialMax) {
       super(field, initialMin, initialMax);
     }

     @Override
     protected final boolean init(LeafReaderContext context) throws IOException {
       sndv = context.reader().getSortedNumericDocValues(field);
       return sndv != null;
     }

     @Override
     protected final boolean hasValue(int doc) throws IOException {
       return sndv.advanceExact(doc);
     }

     /** The mean of all values of the field. */
     public final double mean() {
       return mean;
     }

     /** Returns the variance of all values of the field. */
     public final double variance() {
       int count = count();
       return count > 0 ? variance / count : 0;
     }

     /** Returns the stdev of all values of the field. */
     public final double stdev() {
       return Math.sqrt(variance());
     }

     /** Returns the total number of values for this field. */
     public final long valuesCount() {
       return valuesCount;
     }

     /** Returns the sum of values of the field. Note that if the values are large, the {@code sum} might overflow. */
     public abstract T sum();
   }

   /** Holds DocValues statistics for a sorted-numeric field storing {@code long} values. */
   public static final class SortedLongDocValuesStats extends SortedNumericDocValuesStats<Long> {

     // To avoid boxing 'long' to 'Long' while the sum is computed, declare it as private variable.
     private long sum = 0;

     public SortedLongDocValuesStats(String field) {
       super(field, Long.MAX_VALUE, Long.MIN_VALUE);
     }

     @Override
     protected void doAccumulate(int count) throws IOException {
       int numValues = sndv.docValueCount();
       while (numValues-- > 0) {
         long val = sndv.nextValue();
         if (val > max) {
           max = val;
         }
         if (val < min) {
           min = val;
         }
         sum += val;
         double oldMean = mean;
         // for correct "running average computation", increase valuesCount with each value, rather than once before the
         // loop stats.
         ++valuesCount;
         mean += (val - mean) / valuesCount;
         variance += (val - mean) * (val - oldMean);
       }
     }

     @Override
     public Long sum() {
       return sum;
     }
   }

   /** Holds DocValues statistics for a sorted-numeric field storing {@code double} values. */
   public static final class SortedDoubleDocValuesStats extends SortedNumericDocValuesStats<Double> {

     // To avoid boxing 'double' to 'Double' while the sum is computed, declare it as private variable.
     private double sum = 0;

     public SortedDoubleDocValuesStats(String field) {
       super(field, Double.MAX_VALUE, Double.MIN_VALUE);
     }

     @Override
     protected void doAccumulate(int count) throws IOException {
       int numValues = sndv.docValueCount();
       while (numValues-- > 0) {
         double val = Double.longBitsToDouble(sndv.nextValue());
         if (Double.compare(val, max) > 0) {
           max = val;
         }
         if (Double.compare(val, min) < 0) {
           min = val;
         }
         sum += val;
         double oldMean = mean;
         // for correct "running average computation", increase valuesCount with each value, rather than once before the
         // loop stats.
         ++valuesCount;
         mean += (val - mean) / valuesCount;
         variance += (val - mean) * (val - oldMean);
       }
     }

     @Override
     public Double sum() {
       return sum;
     }
   }

   private static BytesRef copyFrom(BytesRef src, BytesRef dest) {
     if (dest == null) {
       return BytesRef.deepCopyOf(src);
     }

     dest.bytes = ArrayUtil.grow(dest.bytes, src.length);
     System.arraycopy(src.bytes, src.offset, dest.bytes, 0, src.length);
     dest.offset = 0;
     dest.length = src.length;
     return dest;
   }

   /** Holds statistics for a sorted DocValues field. */
   public static class SortedDocValuesStats extends DocValuesStats<BytesRef> {

     protected SortedDocValues sdv;

     protected SortedDocValuesStats(String field) {
       super(field, null, null);
     }

     @Override
     protected final boolean init(LeafReaderContext context) throws IOException {
       sdv = context.reader().getSortedDocValues(field);
       return sdv != null;
     }

     @Override
     protected final boolean hasValue(int doc) throws IOException {
       return sdv.advanceExact(doc);
     }

     @Override
     protected void doAccumulate(int count) throws IOException {
       BytesRef val = sdv.binaryValue();
       if (max == null || val.compareTo(max) > 0) {
         max = copyFrom(val, max);
       }
       if (min == null || val.compareTo(min) < 0) {
         min = copyFrom(val, min);
       }
     }
   }

   /** Holds statistics for a sorted-set DocValues field. */
   public static class SortedSetDocValuesStats extends DocValuesStats<BytesRef> {

     protected SortedSetDocValues ssdv;

     protected SortedSetDocValuesStats(String field) {
       super(field, null, null);
     }

     @Override
     protected final boolean init(LeafReaderContext context) throws IOException {
       ssdv = context.reader().getSortedSetDocValues(field);
       return ssdv != null;
     }

     @Override
     protected final boolean hasValue(int doc) throws IOException {
       return ssdv.advanceExact(doc);
     }

     @Override
     protected void doAccumulate(int count) throws IOException {
       long ord;
       while ((ord = ssdv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
         BytesRef val = ssdv.lookupOrd(ord);
         if (max == null || val.compareTo(max) > 0) {
           max = copyFrom(val, max);
         }
         if (min == null || val.compareTo(min) < 0) {
           min = copyFrom(val, min);
         }
       }
     }
   }

 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.lucene.search;

	import java.io.IOException;

	import org.apache.lucene.index.LeafReaderContext;
	import org.apache.lucene.index.NumericDocValues;
	import org.apache.lucene.index.SortedDocValues;
	import org.apache.lucene.index.SortedNumericDocValues;
	import org.apache.lucene.index.SortedSetDocValues;
	import org.apache.lucene.util.ArrayUtil;
	import org.apache.lucene.util.BytesRef;

	/** Holds statistics for a DocValues field. */
	public abstract class DocValuesStats<T> {

	private int missing = 0;
	private int count = 0;

	protected final String field;

	protected T min;
	protected T max;

	protected DocValuesStats(String field, T initialMin, T initialMax) {
	this.field = field;
	this.min = initialMin;
	this.max = initialMax;
	}

	/**
	* Called after {@link #accumulate(int)} was processed and verified that the document has a value for
	* the field. Implementations should update the statistics based on the value of the current document.
	*
	* @param count
	* the updated number of documents with value for this field.
	*/
	protected abstract void doAccumulate(int count) throws IOException;

	/**
	* Initializes this object with the given reader context. Returns whether stats can be computed for this segment (i.e.
	* it does have the requested DocValues field).
	*/
	protected abstract boolean init(LeafReaderContext context) throws IOException;

	/** Returns whether the given document has a value for the requested DocValues field. */
	protected abstract boolean hasValue(int doc) throws IOException;

	final void accumulate(int doc) throws IOException {
	if (hasValue(doc)) {
	++count;
	doAccumulate(count);
	} else {
	++missing;
	}
	}

	final void addMissing() {
	++missing;
	}

	/** The field for which these stats were computed. */
	public final String field() {
	return field;
	}

	/** The number of documents which have a value of the field. */
	public final int count() {
	return count;
	}

	/** The number of documents which do not have a value of the field. */
	public final int missing() {
	return missing;
	}

	/** The minimum value of the field. Undefined when {@link #count()} is zero. */
	public final T min() {
	return min;
	}

	/** The maximum value of the field. Undefined when {@link #count()} is zero. */
	public final T max() {
	return max;
	}

	/** Holds statistics for a numeric DocValues field. */
	public static abstract class NumericDocValuesStats<T extends Number> extends DocValuesStats<T> {

	protected double mean = 0.0;
	protected double variance = 0.0;

	protected NumericDocValues ndv;

	protected NumericDocValuesStats(String field, T initialMin, T initialMax) {
	super(field, initialMin, initialMax);
	}

	@Override
	protected final boolean init(LeafReaderContext context) throws IOException {
	ndv = context.reader().getNumericDocValues(field);
	return ndv != null;
	}

	@Override
	protected final boolean hasValue(int doc) throws IOException {
	return ndv.advanceExact(doc);
	}

	/** The mean of all values of the field. */
	public final double mean() {
	return mean;
	}

	/** Returns the variance of all values of the field. */
	public final double variance() {
	int count = count();
	return count > 0 ? variance / count : 0;
	}

	/** Returns the stdev of all values of the field. */
	public final double stdev() {
	return Math.sqrt(variance());
	}

	/** Returns the sum of values of the field. Note that if the values are large, the {@code sum} might overflow. */
	public abstract T sum();
	}

	/** Holds DocValues statistics for a numeric field storing {@code long} values. */
	public static final class LongDocValuesStats extends NumericDocValuesStats<Long> {

	// To avoid boxing 'long' to 'Long' while the sum is computed, declare it as private variable.
	private long sum = 0;

	public LongDocValuesStats(String field) {
	super(field, Long.MAX_VALUE, Long.MIN_VALUE);
	}

	@Override
	protected void doAccumulate(int count) throws IOException {
	long val = ndv.longValue();
	if (val > max) {
	max = val;
	}
	if (val < min) {
	min = val;
	}
	sum += val;
	double oldMean = mean;
	mean += (val - mean) / count;
	variance += (val - mean) * (val - oldMean);
	}

	@Override
	public Long sum() {
	return sum;
	}
	}

	/** Holds DocValues statistics for a numeric field storing {@code double} values. */
	public static final class DoubleDocValuesStats extends NumericDocValuesStats<Double> {

	// To avoid boxing 'double' to 'Double' while the sum is computed, declare it as private variable.
	private double sum = 0;

	public DoubleDocValuesStats(String field) {
	super(field, Double.MAX_VALUE, Double.MIN_VALUE);
	}

	@Override
	protected void doAccumulate(int count) throws IOException {
	double val = Double.longBitsToDouble(ndv.longValue());
	if (Double.compare(val, max) > 0) {
	max = val;
	}
	if (Double.compare(val, min) < 0) {
	min = val;
	}
	sum += val;
	double oldMean = mean;
	mean += (val - mean) / count;
	variance += (val - mean) * (val - oldMean);
	}

	@Override
	public Double sum() {
	return sum;
	}
	}

	/** Holds statistics for a sorted-numeric DocValues field. */
	public static abstract class SortedNumericDocValuesStats<T extends Number> extends DocValuesStats<T> {

	protected long valuesCount = 0;
	protected double mean = 0.0;
	protected double variance = 0.0;

	protected SortedNumericDocValues sndv;

	protected SortedNumericDocValuesStats(String field, T initialMin, T initialMax) {
	super(field, initialMin, initialMax);
	}

	@Override
	protected final boolean init(LeafReaderContext context) throws IOException {
	sndv = context.reader().getSortedNumericDocValues(field);
	return sndv != null;
	}

	@Override
	protected final boolean hasValue(int doc) throws IOException {
	return sndv.advanceExact(doc);
	}

	/** The mean of all values of the field. */
	public final double mean() {
	return mean;
	}

	/** Returns the variance of all values of the field. */
	public final double variance() {
	int count = count();
	return count > 0 ? variance / count : 0;
	}

	/** Returns the stdev of all values of the field. */
	public final double stdev() {
	return Math.sqrt(variance());
	}

	/** Returns the total number of values for this field. */
	public final long valuesCount() {
	return valuesCount;
	}

	/** Returns the sum of values of the field. Note that if the values are large, the {@code sum} might overflow. */
	public abstract T sum();
	}

	/** Holds DocValues statistics for a sorted-numeric field storing {@code long} values. */
	public static final class SortedLongDocValuesStats extends SortedNumericDocValuesStats<Long> {

	// To avoid boxing 'long' to 'Long' while the sum is computed, declare it as private variable.
	private long sum = 0;

	public SortedLongDocValuesStats(String field) {
	super(field, Long.MAX_VALUE, Long.MIN_VALUE);
	}

	@Override
	protected void doAccumulate(int count) throws IOException {
	int numValues = sndv.docValueCount();
	while (numValues-- > 0) {
	long val = sndv.nextValue();
	if (val > max) {
	max = val;
	}
	if (val < min) {
	min = val;
	}
	sum += val;
	double oldMean = mean;
	// for correct "running average computation", increase valuesCount with each value, rather than once before the
	// loop stats.
	++valuesCount;
	mean += (val - mean) / valuesCount;
	variance += (val - mean) * (val - oldMean);
	}
	}

	@Override
	public Long sum() {
	return sum;
	}
	}

	/** Holds DocValues statistics for a sorted-numeric field storing {@code double} values. */
	public static final class SortedDoubleDocValuesStats extends SortedNumericDocValuesStats<Double> {

	// To avoid boxing 'double' to 'Double' while the sum is computed, declare it as private variable.
	private double sum = 0;

	public SortedDoubleDocValuesStats(String field) {
	super(field, Double.MAX_VALUE, Double.MIN_VALUE);
	}

	@Override
	protected void doAccumulate(int count) throws IOException {
	int numValues = sndv.docValueCount();
	while (numValues-- > 0) {
	double val = Double.longBitsToDouble(sndv.nextValue());
	if (Double.compare(val, max) > 0) {
	max = val;
	}
	if (Double.compare(val, min) < 0) {
	min = val;
	}
	sum += val;
	double oldMean = mean;
	// for correct "running average computation", increase valuesCount with each value, rather than once before the
	// loop stats.
	++valuesCount;
	mean += (val - mean) / valuesCount;
	variance += (val - mean) * (val - oldMean);
	}
	}

	@Override
	public Double sum() {
	return sum;
	}
	}

	private static BytesRef copyFrom(BytesRef src, BytesRef dest) {
	if (dest == null) {
	return BytesRef.deepCopyOf(src);
	}

	dest.bytes = ArrayUtil.grow(dest.bytes, src.length);
	System.arraycopy(src.bytes, src.offset, dest.bytes, 0, src.length);
	dest.offset = 0;
	dest.length = src.length;
	return dest;
	}

	/** Holds statistics for a sorted DocValues field. */
	public static class SortedDocValuesStats extends DocValuesStats<BytesRef> {

	protected SortedDocValues sdv;

	protected SortedDocValuesStats(String field) {
	super(field, null, null);
	}

	@Override
	protected final boolean init(LeafReaderContext context) throws IOException {
	sdv = context.reader().getSortedDocValues(field);
	return sdv != null;
	}

	@Override
	protected final boolean hasValue(int doc) throws IOException {
	return sdv.advanceExact(doc);
	}

	@Override
	protected void doAccumulate(int count) throws IOException {
	BytesRef val = sdv.binaryValue();
	if (max == null \|\| val.compareTo(max) > 0) {
	max = copyFrom(val, max);
	}
	if (min == null \|\| val.compareTo(min) < 0) {
	min = copyFrom(val, min);
	}
	}
	}

	/** Holds statistics for a sorted-set DocValues field. */
	public static class SortedSetDocValuesStats extends DocValuesStats<BytesRef> {

	protected SortedSetDocValues ssdv;

	protected SortedSetDocValuesStats(String field) {
	super(field, null, null);
	}

	@Override
	protected final boolean init(LeafReaderContext context) throws IOException {
	ssdv = context.reader().getSortedSetDocValues(field);
	return ssdv != null;
	}

	@Override
	protected final boolean hasValue(int doc) throws IOException {
	return ssdv.advanceExact(doc);
	}

	@Override
	protected void doAccumulate(int count) throws IOException {
	long ord;
	while ((ord = ssdv.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
	BytesRef val = ssdv.lookupOrd(ord);
	if (max == null \|\| val.compareTo(max) > 0) {
	max = copyFrom(val, max);
	}
	if (min == null \|\| val.compareTo(min) < 0) {
	min = copyFrom(val, min);
	}
	}
	}
	}

	}