lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.search.similarities;


import java.io.IOException;
import java.util.Arrays;
import java.util.Random;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.Version;

public class TestClassicSimilarity extends BaseSimilarityTestCase {
  private Directory directory;
  private IndexReader indexReader;
  private IndexSearcher indexSearcher;

  @Override
  public void setUp() throws Exception {
    super.setUp();
    directory = newDirectory();
    try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig())) {
      Document document = new Document();
      document.add(new StringField("test", "hit", Store.NO));
      indexWriter.addDocument(document);
      indexWriter.commit();
    }
    indexReader = DirectoryReader.open(directory);
    indexSearcher = newSearcher(indexReader);
    indexSearcher.setSimilarity(new ClassicSimilarity());
  }

  @Override
  public void tearDown() throws Exception {
    IOUtils.close(indexReader, directory);
    super.tearDown();
  }

  public void testHit() throws IOException {
    Query query = new TermQuery(new Term("test", "hit"));
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertTrue(topDocs.scoreDocs[0].score != 0);
  }

  public void testMiss() throws IOException {
    Query query = new TermQuery(new Term("test", "miss"));
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(0, topDocs.totalHits.value);
  }

  public void testEmpty() throws IOException {
    Query query = new TermQuery(new Term("empty", "miss"));
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(0, topDocs.totalHits.value);
  }

  public void testBQHit() throws IOException {
    Query query = new BooleanQuery.Builder()
      .add(new TermQuery(new Term("test", "hit")), Occur.SHOULD)
      .build();
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertTrue(topDocs.scoreDocs[0].score != 0);
  }

  public void testBQHitOrMiss() throws IOException {
    Query query = new BooleanQuery.Builder()
      .add(new TermQuery(new Term("test", "hit")), Occur.SHOULD)
      .add(new TermQuery(new Term("test", "miss")), Occur.SHOULD)
      .build();
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertTrue(topDocs.scoreDocs[0].score != 0);
  }

  public void testBQHitOrEmpty() throws IOException {
    Query query = new BooleanQuery.Builder()
      .add(new TermQuery(new Term("test", "hit")), Occur.SHOULD)
      .add(new TermQuery(new Term("empty", "miss")), Occur.SHOULD)
      .build();
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertTrue(topDocs.scoreDocs[0].score != 0);
  }

  public void testDMQHit() throws IOException {
    Query query = new DisjunctionMaxQuery(
      Arrays.asList(
        new TermQuery(new Term("test", "hit"))),
      0);
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertTrue(topDocs.scoreDocs[0].score != 0);
  }

  public void testDMQHitOrMiss() throws IOException {
    Query query = new DisjunctionMaxQuery(
      Arrays.asList(
        new TermQuery(new Term("test", "hit")),
        new TermQuery(new Term("test", "miss"))),
      0);
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertTrue(topDocs.scoreDocs[0].score != 0);
  }

  public void testDMQHitOrEmpty() throws IOException {
    Query query = new DisjunctionMaxQuery(
      Arrays.asList(
        new TermQuery(new Term("test", "hit")),
        new TermQuery(new Term("empty", "miss"))),
      0);
    TopDocs topDocs = indexSearcher.search(query, 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertTrue(topDocs.scoreDocs[0].score != 0);
  }
  
  public void testSaneNormValues() throws IOException {
    ClassicSimilarity sim = new ClassicSimilarity();
    TFIDFSimilarity.TFIDFScorer stats = (TFIDFSimilarity.TFIDFScorer) sim.scorer(1f, indexSearcher.collectionStatistics("test"));
    for (int i = 0; i < 256; i++) {
      float boost = stats.normTable[i];
      assertFalse("negative boost: " + boost + ", byte=" + i, boost < 0.0f);
      assertFalse("inf bost: " + boost + ", byte=" + i, Float.isInfinite(boost));
      assertFalse("nan boost for byte=" + i, Float.isNaN(boost));
      if (i > 0) {
        assertTrue("boost is not decreasing: " + boost + ",byte=" + i, boost < stats.normTable[i-1]);
      }
    }
  }

  public void testSameNormsAsBM25() {
    ClassicSimilarity sim1 = new ClassicSimilarity();
    BM25Similarity sim2 = new BM25Similarity();
    sim2.setDiscountOverlaps(true);
    for (int iter = 0; iter < 100; ++iter) {
      final int length = TestUtil.nextInt(random(), 1, 1000);
      final int position = random().nextInt(length);
      final int numOverlaps = random().nextInt(length);
      final int maxTermFrequency = 1;
      final int uniqueTermCount = 1;
      FieldInvertState state = new FieldInvertState(Version.LATEST.major, "foo", IndexOptions.DOCS_AND_FREQS, position, length, numOverlaps, 100, maxTermFrequency, uniqueTermCount);
      assertEquals(
          sim2.computeNorm(state),
          sim1.computeNorm(state),
          0f);
    }
  }

  @Override
  protected Similarity getSimilarity(Random random) {
    return new ClassicSimilarity();
  }
}