lucene/replicator/src/test/org/apache/lucene/replicator/IndexAndTaxonomyRevisionTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.replicator;

import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetField;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.SnapshotDeletionPolicy;
import org.apache.lucene.replicator.IndexAndTaxonomyRevision.SnapshotDirectoryTaxonomyWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
import org.junit.Test;

public class IndexAndTaxonomyRevisionTest extends ReplicatorTestCase {
  
  private Document newDocument(TaxonomyWriter taxoWriter) throws IOException {
    FacetsConfig config = new FacetsConfig();
    Document doc = new Document();
    doc.add(new FacetField("A", "1"));
    return config.build(taxoWriter, doc);
  }
  
  @Test
  public void testNoCommit() throws Exception {
    Directory indexDir = newDirectory();
    IndexWriterConfig conf = new IndexWriterConfig(null);
    conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
    IndexWriter indexWriter = new IndexWriter(indexDir, conf);
    
    Directory taxoDir = newDirectory();
    SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
    // should fail when there are no commits to snapshot
    expectThrows(IllegalStateException.class, () -> {
      new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
    });

    indexWriter.close();
    IOUtils.close(taxoWriter, taxoDir, indexDir);
  }
  
  @Test
  public void testRevisionRelease() throws Exception {
    Directory indexDir = newDirectory();
    IndexWriterConfig conf = new IndexWriterConfig(null);
    conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
    IndexWriter indexWriter = new IndexWriter(indexDir, conf);
    
    Directory taxoDir = newDirectory();
    SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
    try {
      indexWriter.addDocument(newDocument(taxoWriter));
      indexWriter.commit();
      taxoWriter.commit();
      Revision rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
      // releasing that revision should not delete the files
      rev1.release();
      assertTrue(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
      assertTrue(slowFileExists(taxoDir, IndexFileNames.SEGMENTS + "_1"));
      
      rev1 = new IndexAndTaxonomyRevision(indexWriter, taxoWriter); // create revision again, so the files are snapshotted
      indexWriter.addDocument(newDocument(taxoWriter));
      indexWriter.commit();
      taxoWriter.commit();
      assertNotNull(new IndexAndTaxonomyRevision(indexWriter, taxoWriter));
      rev1.release(); // this release should trigger the delete of segments_1
      assertFalse(slowFileExists(indexDir, IndexFileNames.SEGMENTS + "_1"));
      indexWriter.close();
    } finally {
      IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
    }
  }
  
  @Test
  public void testSegmentsFileLast() throws Exception {
    Directory indexDir = newDirectory();
    IndexWriterConfig conf = new IndexWriterConfig(null);
    conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
    IndexWriter indexWriter = new IndexWriter(indexDir, conf);
    
    Directory taxoDir = newDirectory();
    SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
    try {
      indexWriter.addDocument(newDocument(taxoWriter));
      indexWriter.commit();
      taxoWriter.commit();
      Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
      Map<String,List<RevisionFile>> sourceFiles = rev.getSourceFiles();
      assertEquals(2, sourceFiles.size());
      for (List<RevisionFile> files : sourceFiles.values()) {
        String lastFile = files.get(files.size() - 1).fileName;
        assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS));
      }
      indexWriter.close();
    } finally {
      IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
    }
  }
  
  @Test
  public void testOpen() throws Exception {
    Directory indexDir = newDirectory();
    IndexWriterConfig conf = new IndexWriterConfig(null);
    conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
    IndexWriter indexWriter = new IndexWriter(indexDir, conf);
    
    Directory taxoDir = newDirectory();
    SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
    try {
      indexWriter.addDocument(newDocument(taxoWriter));
      indexWriter.commit();
      taxoWriter.commit();
      Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
      for (Entry<String,List<RevisionFile>> e : rev.getSourceFiles().entrySet()) {
        String source = e.getKey();
        @SuppressWarnings("resource") // silly, both directories are closed in the end
        Directory dir = source.equals(IndexAndTaxonomyRevision.INDEX_SOURCE) ? indexDir : taxoDir;
        for (RevisionFile file : e.getValue()) {
          IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
          InputStream in = rev.open(source, file.fileName);
          assertEquals(src.length(), in.available());
          byte[] srcBytes = new byte[(int) src.length()];
          byte[] inBytes = new byte[(int) src.length()];
          int offset = 0;
          if (random().nextBoolean()) {
            int skip = random().nextInt(10);
            if (skip >= src.length()) {
              skip = 0;
            }
            in.skip(skip);
            src.seek(skip);
            offset = skip;
          }
          src.readBytes(srcBytes, offset, srcBytes.length - offset);
          in.read(inBytes, offset, inBytes.length - offset);
          assertArrayEquals(srcBytes, inBytes);
          IOUtils.close(src, in);
        }
      }
      indexWriter.close();
    } finally {
      IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
    }
  }
  
}