lucene/replicator/src/java/org/apache/lucene/replicator/nrt/ReplicaFileDeleter.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.replicator.nrt;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;

// TODO: can we factor/share with IFD: this is doing exactly the same thing, but on the replica side

class ReplicaFileDeleter {
  private final Map<String,Integer> refCounts = new HashMap<String,Integer>();
  private final Directory dir;
  private final Node node;

  public ReplicaFileDeleter(Node node, Directory dir) throws IOException {
    this.dir = dir;
    this.node = node;
  }

  /** Used only by asserts: returns true if the file exists
   *  (can be opened), false if it cannot be opened, and
   *  (unlike Java's File.exists) throws IOException if
   *  there's some unexpected error. */
  private static boolean slowFileExists(Directory dir, String fileName) throws IOException {
    try {
      dir.openInput(fileName, IOContext.DEFAULT).close();
      return true;
    } catch (NoSuchFileException | FileNotFoundException e) {
      return false;
    }
  }

  public synchronized void incRef(Collection<String> fileNames) throws IOException {
    for(String fileName : fileNames) {

      assert slowFileExists(dir, fileName): "file " + fileName + " does not exist!";

      Integer curCount = refCounts.get(fileName);
      if (curCount == null) {
        refCounts.put(fileName, 1);
      } else {
        refCounts.put(fileName, curCount.intValue() + 1);
      }
    }
  }

  public synchronized void decRef(Collection<String> fileNames) throws IOException {
    Set<String> toDelete = new HashSet<>();
    for(String fileName : fileNames) {
      Integer curCount = refCounts.get(fileName);
      assert curCount != null: "fileName=" + fileName;
      assert curCount.intValue() > 0;
      if (curCount.intValue() == 1) {
        refCounts.remove(fileName);
        toDelete.add(fileName);
      } else {
        refCounts.put(fileName, curCount.intValue() - 1);
      }
    }

    delete(toDelete);

    // TODO: this local IR could incRef files here, like we do now with IW's NRT readers ... then we can assert this again:

    // we can't assert this, e.g a search can be running when we switch to a new NRT point, holding a previous IndexReader still open for
    // a bit:
    /*
    // We should never attempt deletion of a still-open file:
    Set<String> delOpen = ((MockDirectoryWrapper) dir).getOpenDeletedFiles();
    if (delOpen.isEmpty() == false) {
      node.message("fail: we tried to delete these still-open files: " + delOpen);
      throw new AssertionError("we tried to delete these still-open files: " + delOpen);
    }
    */
  }

  private synchronized void delete(Collection<String> toDelete) throws IOException {
    if (Node.VERBOSE_FILES) {
      node.message("now delete " + toDelete.size() + " files: " + toDelete);
    }

    // First pass: delete any segments_N files.  We do these first to be certain stale commit points are removed
    // before we remove any files they reference, in case we crash right now:
    for (String fileName : toDelete) {
      assert refCounts.containsKey(fileName) == false;
      if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
        delete(fileName);
      }
    }

    // Only delete other files if we were able to remove the segments_N files; this way we never
    // leave a corrupt commit in the index even in the presense of virus checkers:
    for(String fileName : toDelete) {
      assert refCounts.containsKey(fileName) == false;
      if (fileName.startsWith(IndexFileNames.SEGMENTS) == false) {
        delete(fileName);
      }
    }

  }

  private synchronized void delete(String fileName) throws IOException {
    if (Node.VERBOSE_FILES) {
      node.message("file " + fileName + ": now delete");
    }
    dir.deleteFile(fileName);
  }

  public synchronized Integer getRefCount(String fileName) {
    return refCounts.get(fileName);
  }

  public synchronized void deleteIfNoRef(String fileName) throws IOException {
    if (refCounts.containsKey(fileName) == false) {
      deleteNewFile(fileName);
    }
  }

  public synchronized void deleteNewFile(String fileName) throws IOException {
    delete(fileName);
  }

  /*
  public synchronized Set<String> getPending() {
    return new HashSet<String>(pending);
  }
  */

  public synchronized void deleteUnknownFiles(String segmentsFileName) throws IOException {
    Set<String> toDelete = new HashSet<>();
    for(String fileName : dir.listAll()) {
      if (refCounts.containsKey(fileName) == false &&
          fileName.equals("write.lock") == false &&
          fileName.equals(segmentsFileName) == false) {
        node.message("will delete unknown file \"" + fileName + "\"");
        toDelete.add(fileName);
      }
    }

    delete(toDelete);
  }
}