hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java - hbase - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.hadoop.hbase.util;

 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;

 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 /**
  * Test helper for testing archiving of HFiles
  */
 public class HFileArchiveTestingUtil {

   private static final Logger LOG = LoggerFactory.getLogger(HFileArchiveTestingUtil.class);

   private HFileArchiveTestingUtil() {
     // NOOP private ctor since this is just a utility class
   }

   public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived,
       FileSystem fs, boolean hasTimedBackup) {

     List<List<String>> lists = getFileLists(previous, archived);
     List<String> original = lists.get(0);
     Collections.sort(original);

     List<String> currentFiles = lists.get(1);
     Collections.sort(currentFiles);

     List<String> backedup = lists.get(2);
     Collections.sort(backedup);

     // check the backed up files versus the current (should match up, less the
     // backup time in the name)
     if (!hasTimedBackup == (backedup.size() > 0)) {
       LOG.debug("backedup files doesn't match expected.");
       return false;
     }
     String msg = null;
     if (hasTimedBackup) {
       msg = assertArchiveEquality(original, backedup);
       if (msg != null) {
         LOG.debug(msg);
         return false;
       }
     }
     msg = assertArchiveEquality(original, currentFiles);
     if (msg != null) {
       LOG.debug(msg);
       return false;
     }
     return true;
   }

   /**
    * Compare the archived files to the files in the original directory
    * @param expected original files that should have been archived
    * @param actual files that were archived
    * @param fs filessystem on which the archiving took place
    * @throws IOException
    */
   public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
       FileSystem fs) throws IOException {
     assertArchiveEqualToOriginal(expected, actual, fs, false);
   }

   /**
    * Compare the archived files to the files in the original directory
    * @param expected original files that should have been archived
    * @param actual files that were archived
    * @param fs {@link FileSystem} on which the archiving took place
    * @param hasTimedBackup <tt>true</tt> if we expect to find an archive backup directory with a
    *          copy of the files in the archive directory (and the original files).
    * @throws IOException
    */
   public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
       FileSystem fs, boolean hasTimedBackup) throws IOException {

     List<List<String>> lists = getFileLists(expected, actual);
     List<String> original = lists.get(0);
     Collections.sort(original);

     List<String> currentFiles = lists.get(1);
     Collections.sort(currentFiles);

     List<String> backedup = lists.get(2);
     Collections.sort(backedup);

     // check the backed up files versus the current (should match up, less the
     // backup time in the name)
     assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup,
       backedup.size() > 0);
     String msg = null;
     if (hasTimedBackup) {
       assertArchiveEquality(original, backedup);
       assertNull(msg, msg);
     }

     // do the rest of the comparison
     msg = assertArchiveEquality(original, currentFiles);
     assertNull(msg, msg);
   }

   private static String assertArchiveEquality(List<String> expected, List<String> archived) {
     String compare = compareFileLists(expected, archived);
     if (!(expected.size() == archived.size())) return "Not the same number of current files\n"
         + compare;
     if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare;
     return null;
   }

   /**
    * @return &lt;expected, gotten, backup&gt;, where each is sorted
    */
   private static List<List<String>> getFileLists(FileStatus[] previous, FileStatus[] archived) {
     List<List<String>> files = new ArrayList<>(3);

     // copy over the original files
     List<String> originalFileNames = convertToString(previous);
     files.add(originalFileNames);

     List<String> currentFiles = new ArrayList<>(previous.length);
     List<FileStatus> backedupFiles = new ArrayList<>(previous.length);
     for (FileStatus f : archived) {
       String name = f.getPath().getName();
       // if the file has been backed up
       if (name.contains(".")) {
         Path parent = f.getPath().getParent();
         String shortName = name.split("[.]")[0];
         Path modPath = new Path(parent, shortName);
         FileStatus file = new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(),
             f.getBlockSize(), f.getModificationTime(), modPath);
         backedupFiles.add(file);
       } else {
         // otherwise, add it to the list to compare to the original store files
         currentFiles.add(name);
       }
     }

     files.add(currentFiles);
     files.add(convertToString(backedupFiles));
     return files;
   }

   private static List<String> convertToString(FileStatus[] files) {
     return convertToString(Arrays.asList(files));
   }

   private static List<String> convertToString(List<FileStatus> files) {
     List<String> originalFileNames = new ArrayList<>(files.size());
     for (FileStatus f : files) {
       originalFileNames.add(f.getPath().getName());
     }
     return originalFileNames;
   }

   /* Get a pretty representation of the differences */
   private static String compareFileLists(List<String> expected, List<String> gotten) {
     StringBuilder sb = new StringBuilder("Expected (" + expected.size() + "): \t\t Gotten ("
         + gotten.size() + "):\n");
     List<String> notFound = new ArrayList<>();
     for (String s : expected) {
       if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n");
       else notFound.add(s);
     }
     sb.append("Not Found:\n");
     for (String s : notFound) {
       sb.append(s + "\n");
     }
     sb.append("\nExtra:\n");
     for (String s : gotten) {
       if (!expected.contains(s)) sb.append(s + "\n");
     }
     return sb.toString();
   }

   /**
    * Helper method to get the archive directory for the specified region
    * @param conf {@link Configuration} to check for the name of the archive directory
    * @param region region that is being archived
    * @return {@link Path} to the archive directory for the given region
    */
   public static Path getRegionArchiveDir(Configuration conf, HRegion region) throws IOException {
     return HFileArchiveUtil.getRegionArchiveDir(CommonFSUtils.getRootDir(conf),
       region.getTableDescriptor().getTableName(), region.getRegionInfo().getEncodedName());
   }

   /**
    * Helper method to get the store archive directory for the specified region
    * @param conf {@link Configuration} to check for the name of the archive directory
    * @param region region that is being archived
    * @param store store that is archiving files
    * @return {@link Path} to the store archive directory for the given region
    */
   public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store)
       throws IOException {
     return HFileArchiveUtil.getStoreArchivePath(conf, region.getRegionInfo(),
         region.getRegionFileSystem().getTableDir(), store.getColumnFamilyDescriptor().getName());
   }

   public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,
       byte[] storeName) throws IOException {
     byte[] table = Bytes.toBytes(tableName);
     // get the RS and region serving our table
     List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
     HRegion region = servingRegions.get(0);

     // check that we actually have some store files that were archived
     Store store = region.getStore(storeName);
     return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
   }
 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.hadoop.hbase.util;

	import static org.junit.Assert.assertEquals;
	import static org.junit.Assert.assertNull;

	import java.io.IOException;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.Collections;
	import java.util.List;
	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.FileStatus;
	import org.apache.hadoop.fs.FileSystem;
	import org.apache.hadoop.fs.Path;
	import org.apache.hadoop.hbase.HBaseTestingUtility;
	import org.apache.hadoop.hbase.regionserver.HRegion;
	import org.apache.hadoop.hbase.regionserver.Store;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	/**
	* Test helper for testing archiving of HFiles
	*/
	public class HFileArchiveTestingUtil {

	private static final Logger LOG = LoggerFactory.getLogger(HFileArchiveTestingUtil.class);

	private HFileArchiveTestingUtil() {
	// NOOP private ctor since this is just a utility class
	}

	public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived,
	FileSystem fs, boolean hasTimedBackup) {

	List<List<String>> lists = getFileLists(previous, archived);
	List<String> original = lists.get(0);
	Collections.sort(original);

	List<String> currentFiles = lists.get(1);
	Collections.sort(currentFiles);

	List<String> backedup = lists.get(2);
	Collections.sort(backedup);

	// check the backed up files versus the current (should match up, less the
	// backup time in the name)
	if (!hasTimedBackup == (backedup.size() > 0)) {
	LOG.debug("backedup files doesn't match expected.");
	return false;
	}
	String msg = null;
	if (hasTimedBackup) {
	msg = assertArchiveEquality(original, backedup);
	if (msg != null) {
	LOG.debug(msg);
	return false;
	}
	}
	msg = assertArchiveEquality(original, currentFiles);
	if (msg != null) {
	LOG.debug(msg);
	return false;
	}
	return true;
	}

	/**
	* Compare the archived files to the files in the original directory
	* @param expected original files that should have been archived
	* @param actual files that were archived
	* @param fs filessystem on which the archiving took place
	* @throws IOException
	*/
	public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
	FileSystem fs) throws IOException {
	assertArchiveEqualToOriginal(expected, actual, fs, false);
	}

	/**
	* Compare the archived files to the files in the original directory
	* @param expected original files that should have been archived
	* @param actual files that were archived
	* @param fs {@link FileSystem} on which the archiving took place
	* @param hasTimedBackup <tt>true</tt> if we expect to find an archive backup directory with a
	* copy of the files in the archive directory (and the original files).
	* @throws IOException
	*/
	public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
	FileSystem fs, boolean hasTimedBackup) throws IOException {

	List<List<String>> lists = getFileLists(expected, actual);
	List<String> original = lists.get(0);
	Collections.sort(original);

	List<String> currentFiles = lists.get(1);
	Collections.sort(currentFiles);

	List<String> backedup = lists.get(2);
	Collections.sort(backedup);

	// check the backed up files versus the current (should match up, less the
	// backup time in the name)
	assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup,
	backedup.size() > 0);
	String msg = null;
	if (hasTimedBackup) {
	assertArchiveEquality(original, backedup);
	assertNull(msg, msg);
	}

	// do the rest of the comparison
	msg = assertArchiveEquality(original, currentFiles);
	assertNull(msg, msg);
	}

	private static String assertArchiveEquality(List<String> expected, List<String> archived) {
	String compare = compareFileLists(expected, archived);
	if (!(expected.size() == archived.size())) return "Not the same number of current files\n"
	+ compare;
	if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare;
	return null;
	}

	/**
	* @return <expected, gotten, backup>, where each is sorted
	*/
	private static List<List<String>> getFileLists(FileStatus[] previous, FileStatus[] archived) {
	List<List<String>> files = new ArrayList<>(3);

	// copy over the original files
	List<String> originalFileNames = convertToString(previous);
	files.add(originalFileNames);

	List<String> currentFiles = new ArrayList<>(previous.length);
	List<FileStatus> backedupFiles = new ArrayList<>(previous.length);
	for (FileStatus f : archived) {
	String name = f.getPath().getName();
	// if the file has been backed up
	if (name.contains(".")) {
	Path parent = f.getPath().getParent();
	String shortName = name.split("[.]")[0];
	Path modPath = new Path(parent, shortName);
	FileStatus file = new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(),
	f.getBlockSize(), f.getModificationTime(), modPath);
	backedupFiles.add(file);
	} else {
	// otherwise, add it to the list to compare to the original store files
	currentFiles.add(name);
	}
	}

	files.add(currentFiles);
	files.add(convertToString(backedupFiles));
	return files;
	}

	private static List<String> convertToString(FileStatus[] files) {
	return convertToString(Arrays.asList(files));
	}

	private static List<String> convertToString(List<FileStatus> files) {
	List<String> originalFileNames = new ArrayList<>(files.size());
	for (FileStatus f : files) {
	originalFileNames.add(f.getPath().getName());
	}
	return originalFileNames;
	}

	/* Get a pretty representation of the differences */
	private static String compareFileLists(List<String> expected, List<String> gotten) {
	StringBuilder sb = new StringBuilder("Expected (" + expected.size() + "): \t\t Gotten ("
	+ gotten.size() + "):\n");
	List<String> notFound = new ArrayList<>();
	for (String s : expected) {
	if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n");
	else notFound.add(s);
	}
	sb.append("Not Found:\n");
	for (String s : notFound) {
	sb.append(s + "\n");
	}
	sb.append("\nExtra:\n");
	for (String s : gotten) {
	if (!expected.contains(s)) sb.append(s + "\n");
	}
	return sb.toString();
	}

	/**
	* Helper method to get the archive directory for the specified region
	* @param conf {@link Configuration} to check for the name of the archive directory
	* @param region region that is being archived
	* @return {@link Path} to the archive directory for the given region
	*/
	public static Path getRegionArchiveDir(Configuration conf, HRegion region) throws IOException {
	return HFileArchiveUtil.getRegionArchiveDir(CommonFSUtils.getRootDir(conf),
	region.getTableDescriptor().getTableName(), region.getRegionInfo().getEncodedName());
	}

	/**
	* Helper method to get the store archive directory for the specified region
	* @param conf {@link Configuration} to check for the name of the archive directory
	* @param region region that is being archived
	* @param store store that is archiving files
	* @return {@link Path} to the store archive directory for the given region
	*/
	public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store)
	throws IOException {
	return HFileArchiveUtil.getStoreArchivePath(conf, region.getRegionInfo(),
	region.getRegionFileSystem().getTableDir(), store.getColumnFamilyDescriptor().getName());
	}

	public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,
	byte[] storeName) throws IOException {
	byte[] table = Bytes.toBytes(tableName);
	// get the RS and region serving our table
	List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
	HRegion region = servingRegions.get(0);

	// check that we actually have some store files that were archived
	Store store = region.getStore(storeName);
	return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
	}
	}