lucene/test-framework/src/java/org/apache/lucene/util/TestRuleTemporaryFilesCleanup.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.lucene.util;

 import java.io.IOException;
 import java.net.URI;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.spi.FileSystemProvider;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Random;
 import java.util.Set;

 import org.apache.lucene.mockfile.DisableFsyncFS;
 import org.apache.lucene.mockfile.ExtrasFS;
 import org.apache.lucene.mockfile.HandleLimitFS;
 import org.apache.lucene.mockfile.LeakFS;
 import org.apache.lucene.mockfile.ShuffleFS;
 import org.apache.lucene.mockfile.VerboseFS;
 import org.apache.lucene.mockfile.WindowsFS;
 import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
 import org.apache.lucene.util.LuceneTestCase.SuppressFsync;
 import org.apache.lucene.util.LuceneTestCase.SuppressTempFileChecks;

 import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;

 /**
  * Checks and cleans up temporary files.
  *
  * @see LuceneTestCase#createTempDir()
  * @see LuceneTestCase#createTempFile()
  */
 final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
   /**
    * Retry to create temporary file name this many times.
    */
   private static final int TEMP_NAME_RETRY_THRESHOLD = 9999;

   /**
    * Writeable temporary base folder.
    */
   private Path javaTempDir;

   /**
    * Per-test class temporary folder.
    */
   private Path tempDirBase;

   /**
    * Per-test filesystem
    */
   private FileSystem fileSystem;

   /**
    * Suite failure marker.
    */
   private final TestRuleMarkFailure failureMarker;

   /**
    * A queue of temporary resources to be removed after the
    * suite completes.
    * @see #registerToRemoveAfterSuite(Path)
    */
   private final static List<Path> cleanupQueue = new ArrayList<Path>();

   public TestRuleTemporaryFilesCleanup(TestRuleMarkFailure failureMarker) {
     this.failureMarker = failureMarker;
   }

   /**
    * Register temporary folder for removal after the suite completes.
    */
   void registerToRemoveAfterSuite(Path f) {
     assert f != null;

     if (LuceneTestCase.LEAVE_TEMPORARY) {
       System.err.println("INFO: Will leave temporary file: " + f.toAbsolutePath());
       return;
     }

     synchronized (cleanupQueue) {
       cleanupQueue.add(f);
     }
   }

   @Override
   protected void before() throws Throwable {
     super.before();

     assert tempDirBase == null;
     fileSystem = initializeFileSystem();
     javaTempDir = initializeJavaTempDir();
   }

   // os/config-independent limit for too many open files
   // TODO: can we make this lower?
   private static final int MAX_OPEN_FILES = 2048;

   private boolean allowed(Set<String> avoid, Class<? extends FileSystemProvider> clazz) {
     if (avoid.contains("*") || avoid.contains(clazz.getSimpleName())) {
       return false;
     } else {
       return true;
     }
   }

   private FileSystem initializeFileSystem() {
     Class<?> targetClass = RandomizedContext.current().getTargetClass();
     Set<String> avoid = new HashSet<>();
     if (targetClass.isAnnotationPresent(SuppressFileSystems.class)) {
       SuppressFileSystems a = targetClass.getAnnotation(SuppressFileSystems.class);
       avoid.addAll(Arrays.asList(a.value()));
     }
     FileSystem fs = FileSystems.getDefault();
     if (LuceneTestCase.VERBOSE && allowed(avoid, VerboseFS.class)) {
       fs = new VerboseFS(fs, new TestRuleSetupAndRestoreClassEnv.ThreadNameFixingPrintStreamInfoStream(System.out)).getFileSystem(null);
     }

     Random random = RandomizedContext.current().getRandom();

     // speed up tests by omitting actual fsync calls to the hardware most of the time.
     if (targetClass.isAnnotationPresent(SuppressFsync.class) || random.nextInt(100) > 0) {
       if (allowed(avoid, DisableFsyncFS.class)) {
         fs = new DisableFsyncFS(fs).getFileSystem(null);
       }
     }

     // impacts test reproducibility across platforms.
     if (random.nextInt(100) > 0) {
       if (allowed(avoid, ShuffleFS.class)) {
         fs = new ShuffleFS(fs, random.nextLong()).getFileSystem(null);
       }
     }

     // otherwise, wrap with mockfilesystems for additional checks. some
     // of these have side effects (e.g. concurrency) so it doesn't always happen.
     if (random.nextInt(10) > 0) {
       if (allowed(avoid, LeakFS.class)) {
         fs = new LeakFS(fs).getFileSystem(null);
       }
       if (allowed(avoid, HandleLimitFS.class)) {
         fs = new HandleLimitFS(fs, MAX_OPEN_FILES).getFileSystem(null);
       }
       // windows is currently slow
       if (random.nextInt(10) == 0) {
         // don't try to emulate windows on windows: they don't get along
         if (!Constants.WINDOWS && allowed(avoid, WindowsFS.class)) {
           fs = new WindowsFS(fs).getFileSystem(null);
         }
       }
       if (allowed(avoid, ExtrasFS.class)) {
         fs = new ExtrasFS(fs, random.nextInt(4) == 0, random.nextBoolean()).getFileSystem(null);
       }
     }
     if (LuceneTestCase.VERBOSE) {
       System.out.println("filesystem: " + fs.provider());
     }

     return fs.provider().getFileSystem(URI.create("file:///"));
   }

   private Path initializeJavaTempDir() throws IOException {
     Path javaTempDir = fileSystem.getPath(System.getProperty("tempDir", System.getProperty("java.io.tmpdir")));

     Files.createDirectories(javaTempDir);

     assert Files.isDirectory(javaTempDir) &&
            Files.isWritable(javaTempDir);

     return javaTempDir.toRealPath();
   }

   @Override
   protected void afterAlways(List<Throwable> errors) throws Throwable {
     // Drain cleanup queue and clear it.
     final Path [] everything;
     final String tempDirBasePath;
     synchronized (cleanupQueue) {
       tempDirBasePath = (tempDirBase != null ? tempDirBase.toAbsolutePath().toString() : null);
       tempDirBase = null;

       Collections.reverse(cleanupQueue);
       everything = new Path [cleanupQueue.size()];
       cleanupQueue.toArray(everything);
       cleanupQueue.clear();
     }

     // Only check and throw an IOException on un-removable files if the test
     // was successful. Otherwise just report the path of temporary files
     // and leave them there.
     if (failureMarker.wasSuccessful()) {

       try {
         IOUtils.rm(everything);
       } catch (IOException e) {
         Class<?> suiteClass = RandomizedContext.current().getTargetClass();
         if (suiteClass.isAnnotationPresent(SuppressTempFileChecks.class)) {
           System.err.println("WARNING: Leftover undeleted temporary files (bugUrl: "
               + suiteClass.getAnnotation(SuppressTempFileChecks.class).bugUrl() + "): "
               + e.getMessage());
           return;
         }
         throw e;
       }
       if (fileSystem != FileSystems.getDefault()) {
         fileSystem.close();
       }
     } else {
       if (tempDirBasePath != null) {
         System.err.println("NOTE: leaving temporary files on disk at: " + tempDirBasePath);
       }
     }
   }

   Path getPerTestClassTempDir() {
     if (tempDirBase == null) {
       RandomizedContext ctx = RandomizedContext.current();
       Class<?> clazz = ctx.getTargetClass();
       String prefix = clazz.getName();
       prefix = prefix.replaceFirst("^org.apache.lucene.", "lucene.");
       prefix = prefix.replaceFirst("^org.apache.solr.", "solr.");

       int attempt = 0;
       Path f;
       boolean success = false;
       do {
         if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
           throw new RuntimeException(
               "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
                 + javaTempDir.toAbsolutePath());
         }
         f = javaTempDir.resolve(prefix + "_" + ctx.getRunnerSeedAsString()
               + "-" + String.format(Locale.ENGLISH, "%03d", attempt));
         try {
           Files.createDirectory(f);
           success = true;
         } catch (IOException ignore) {}
       } while (!success);

       tempDirBase = f;
       registerToRemoveAfterSuite(tempDirBase);
     }
     return tempDirBase;
   }

   /**
    * @see LuceneTestCase#createTempDir()
    */
   public Path createTempDir(String prefix) {
     Path base = getPerTestClassTempDir();

     int attempt = 0;
     Path f;
     boolean success = false;
     do {
       if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
         throw new RuntimeException(
             "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
               + base.toAbsolutePath());
       }
       f = base.resolve(prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt));
       try {
         Files.createDirectory(f);
         success = true;
       } catch (IOException ignore) {}
     } while (!success);

     registerToRemoveAfterSuite(f);
     return f;
   }

   /**
    * @see LuceneTestCase#createTempFile()
    */
   public Path createTempFile(String prefix, String suffix) throws IOException {
     Path base = getPerTestClassTempDir();

     int attempt = 0;
     Path f;
     boolean success = false;
     do {
       if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
         throw new RuntimeException(
             "Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
               + base.toAbsolutePath());
       }
       f = base.resolve(prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt) + suffix);
       try {
         Files.createFile(f);
         success = true;
       } catch (IOException ignore) {}
     } while (!success);

     registerToRemoveAfterSuite(f);
     return f;
   }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.lucene.util;

	import java.io.IOException;
	import java.net.URI;
	import java.nio.file.FileSystem;
	import java.nio.file.FileSystems;
	import java.nio.file.Files;
	import java.nio.file.Path;
	import java.nio.file.spi.FileSystemProvider;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.Collections;
	import java.util.HashSet;
	import java.util.List;
	import java.util.Locale;
	import java.util.Random;
	import java.util.Set;

	import org.apache.lucene.mockfile.DisableFsyncFS;
	import org.apache.lucene.mockfile.ExtrasFS;
	import org.apache.lucene.mockfile.HandleLimitFS;
	import org.apache.lucene.mockfile.LeakFS;
	import org.apache.lucene.mockfile.ShuffleFS;
	import org.apache.lucene.mockfile.VerboseFS;
	import org.apache.lucene.mockfile.WindowsFS;
	import org.apache.lucene.util.LuceneTestCase.SuppressFileSystems;
	import org.apache.lucene.util.LuceneTestCase.SuppressFsync;
	import org.apache.lucene.util.LuceneTestCase.SuppressTempFileChecks;

	import com.carrotsearch.randomizedtesting.RandomizedContext;
	import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;

	/**
	* Checks and cleans up temporary files.
	*
	* @see LuceneTestCase#createTempDir()
	* @see LuceneTestCase#createTempFile()
	*/
	final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
	/**
	* Retry to create temporary file name this many times.
	*/
	private static final int TEMP_NAME_RETRY_THRESHOLD = 9999;

	/**
	* Writeable temporary base folder.
	*/
	private Path javaTempDir;

	/**
	* Per-test class temporary folder.
	*/
	private Path tempDirBase;

	/**
	* Per-test filesystem
	*/
	private FileSystem fileSystem;

	/**
	* Suite failure marker.
	*/
	private final TestRuleMarkFailure failureMarker;

	/**
	* A queue of temporary resources to be removed after the
	* suite completes.
	* @see #registerToRemoveAfterSuite(Path)
	*/
	private final static List<Path> cleanupQueue = new ArrayList<Path>();

	public TestRuleTemporaryFilesCleanup(TestRuleMarkFailure failureMarker) {
	this.failureMarker = failureMarker;
	}

	/**
	* Register temporary folder for removal after the suite completes.
	*/
	void registerToRemoveAfterSuite(Path f) {
	assert f != null;

	if (LuceneTestCase.LEAVE_TEMPORARY) {
	System.err.println("INFO: Will leave temporary file: " + f.toAbsolutePath());
	return;
	}

	synchronized (cleanupQueue) {
	cleanupQueue.add(f);
	}
	}

	@Override
	protected void before() throws Throwable {
	super.before();

	assert tempDirBase == null;
	fileSystem = initializeFileSystem();
	javaTempDir = initializeJavaTempDir();
	}

	// os/config-independent limit for too many open files
	// TODO: can we make this lower?
	private static final int MAX_OPEN_FILES = 2048;

	private boolean allowed(Set<String> avoid, Class<? extends FileSystemProvider> clazz) {
	if (avoid.contains("*") \|\| avoid.contains(clazz.getSimpleName())) {
	return false;
	} else {
	return true;
	}
	}

	private FileSystem initializeFileSystem() {
	Class<?> targetClass = RandomizedContext.current().getTargetClass();
	Set<String> avoid = new HashSet<>();
	if (targetClass.isAnnotationPresent(SuppressFileSystems.class)) {
	SuppressFileSystems a = targetClass.getAnnotation(SuppressFileSystems.class);
	avoid.addAll(Arrays.asList(a.value()));
	}
	FileSystem fs = FileSystems.getDefault();
	if (LuceneTestCase.VERBOSE && allowed(avoid, VerboseFS.class)) {
	fs = new VerboseFS(fs, new TestRuleSetupAndRestoreClassEnv.ThreadNameFixingPrintStreamInfoStream(System.out)).getFileSystem(null);
	}

	Random random = RandomizedContext.current().getRandom();

	// speed up tests by omitting actual fsync calls to the hardware most of the time.
	if (targetClass.isAnnotationPresent(SuppressFsync.class) \|\| random.nextInt(100) > 0) {
	if (allowed(avoid, DisableFsyncFS.class)) {
	fs = new DisableFsyncFS(fs).getFileSystem(null);
	}
	}

	// impacts test reproducibility across platforms.
	if (random.nextInt(100) > 0) {
	if (allowed(avoid, ShuffleFS.class)) {
	fs = new ShuffleFS(fs, random.nextLong()).getFileSystem(null);
	}
	}

	// otherwise, wrap with mockfilesystems for additional checks. some
	// of these have side effects (e.g. concurrency) so it doesn't always happen.
	if (random.nextInt(10) > 0) {
	if (allowed(avoid, LeakFS.class)) {
	fs = new LeakFS(fs).getFileSystem(null);
	}
	if (allowed(avoid, HandleLimitFS.class)) {
	fs = new HandleLimitFS(fs, MAX_OPEN_FILES).getFileSystem(null);
	}
	// windows is currently slow
	if (random.nextInt(10) == 0) {
	// don't try to emulate windows on windows: they don't get along
	if (!Constants.WINDOWS && allowed(avoid, WindowsFS.class)) {
	fs = new WindowsFS(fs).getFileSystem(null);
	}
	}
	if (allowed(avoid, ExtrasFS.class)) {
	fs = new ExtrasFS(fs, random.nextInt(4) == 0, random.nextBoolean()).getFileSystem(null);
	}
	}
	if (LuceneTestCase.VERBOSE) {
	System.out.println("filesystem: " + fs.provider());
	}

	return fs.provider().getFileSystem(URI.create("file:///"));
	}

	private Path initializeJavaTempDir() throws IOException {
	Path javaTempDir = fileSystem.getPath(System.getProperty("tempDir", System.getProperty("java.io.tmpdir")));

	Files.createDirectories(javaTempDir);

	assert Files.isDirectory(javaTempDir) &&
	Files.isWritable(javaTempDir);

	return javaTempDir.toRealPath();
	}

	@Override
	protected void afterAlways(List<Throwable> errors) throws Throwable {
	// Drain cleanup queue and clear it.
	final Path [] everything;
	final String tempDirBasePath;
	synchronized (cleanupQueue) {
	tempDirBasePath = (tempDirBase != null ? tempDirBase.toAbsolutePath().toString() : null);
	tempDirBase = null;

	Collections.reverse(cleanupQueue);
	everything = new Path [cleanupQueue.size()];
	cleanupQueue.toArray(everything);
	cleanupQueue.clear();
	}

	// Only check and throw an IOException on un-removable files if the test
	// was successful. Otherwise just report the path of temporary files
	// and leave them there.
	if (failureMarker.wasSuccessful()) {

	try {
	IOUtils.rm(everything);
	} catch (IOException e) {
	Class<?> suiteClass = RandomizedContext.current().getTargetClass();
	if (suiteClass.isAnnotationPresent(SuppressTempFileChecks.class)) {
	System.err.println("WARNING: Leftover undeleted temporary files (bugUrl: "
	+ suiteClass.getAnnotation(SuppressTempFileChecks.class).bugUrl() + "): "
	+ e.getMessage());
	return;
	}
	throw e;
	}
	if (fileSystem != FileSystems.getDefault()) {
	fileSystem.close();
	}
	} else {
	if (tempDirBasePath != null) {
	System.err.println("NOTE: leaving temporary files on disk at: " + tempDirBasePath);
	}
	}
	}

	Path getPerTestClassTempDir() {
	if (tempDirBase == null) {
	RandomizedContext ctx = RandomizedContext.current();
	Class<?> clazz = ctx.getTargetClass();
	String prefix = clazz.getName();
	prefix = prefix.replaceFirst("^org.apache.lucene.", "lucene.");
	prefix = prefix.replaceFirst("^org.apache.solr.", "solr.");

	int attempt = 0;
	Path f;
	boolean success = false;
	do {
	if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
	throw new RuntimeException(
	"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
	+ javaTempDir.toAbsolutePath());
	}
	f = javaTempDir.resolve(prefix + "_" + ctx.getRunnerSeedAsString()
	+ "-" + String.format(Locale.ENGLISH, "%03d", attempt));
	try {
	Files.createDirectory(f);
	success = true;
	} catch (IOException ignore) {}
	} while (!success);

	tempDirBase = f;
	registerToRemoveAfterSuite(tempDirBase);
	}
	return tempDirBase;
	}

	/**
	* @see LuceneTestCase#createTempDir()
	*/
	public Path createTempDir(String prefix) {
	Path base = getPerTestClassTempDir();

	int attempt = 0;
	Path f;
	boolean success = false;
	do {
	if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
	throw new RuntimeException(
	"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
	+ base.toAbsolutePath());
	}
	f = base.resolve(prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt));
	try {
	Files.createDirectory(f);
	success = true;
	} catch (IOException ignore) {}
	} while (!success);

	registerToRemoveAfterSuite(f);
	return f;
	}

	/**
	* @see LuceneTestCase#createTempFile()
	*/
	public Path createTempFile(String prefix, String suffix) throws IOException {
	Path base = getPerTestClassTempDir();

	int attempt = 0;
	Path f;
	boolean success = false;
	do {
	if (attempt++ >= TEMP_NAME_RETRY_THRESHOLD) {
	throw new RuntimeException(
	"Failed to get a temporary name too many times, check your temp directory and consider manually cleaning it: "
	+ base.toAbsolutePath());
	}
	f = base.resolve(prefix + "-" + String.format(Locale.ENGLISH, "%03d", attempt) + suffix);
	try {
	Files.createFile(f);
	success = true;
	} catch (IOException ignore) {}
	} while (!success);

	registerToRemoveAfterSuite(f);
	return f;
	}
	}