src/java/org/apache/cassandra/tools/StandaloneSplitter.java - cassandra - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *   http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
  */
 package org.apache.cassandra.tools;

 import java.io.File;
 import java.util.*;
 import java.util.concurrent.TimeUnit;

 import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.commons.cli.*;

 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.SSTableSplitter;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.io.sstable.*;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.Pair;

 import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;

 public class StandaloneSplitter
 {
     public static final int DEFAULT_SSTABLE_SIZE = 50;

     private static final String TOOL_NAME = "sstablessplit";
     private static final String DEBUG_OPTION = "debug";
     private static final String HELP_OPTION = "help";
     private static final String NO_SNAPSHOT_OPTION = "no-snapshot";
     private static final String SIZE_OPTION = "size";

     public static void main(String args[])
     {
         Options options = Options.parseArgs(args);
         Util.initDatabaseDescriptor();

         try
         {
             // load keyspace descriptions.
             Schema.instance.loadFromDisk(false);

             String ksName = null;
             String cfName = null;
             Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
             for (String filename : options.filenames)
             {
                 File file = new File(filename);
                 if (!file.exists()) {
                     System.out.println("Skipping inexisting file " + file);
                     continue;
                 }

                 Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
                 if (pair == null) {
                     System.out.println("Skipping non sstable file " + file);
                     continue;
                 }
                 Descriptor desc = pair.left;

                 if (ksName == null)
                     ksName = desc.ksname;
                 else if (!ksName.equals(desc.ksname))
                     throw new IllegalArgumentException("All sstables must be part of the same keyspace");

                 if (cfName == null)
                     cfName = desc.cfname;
                 else if (!cfName.equals(desc.cfname))
                     throw new IllegalArgumentException("All sstables must be part of the same table");

                 Set<Component> components = new HashSet<Component>(Arrays.asList(new Component[]{
                     Component.DATA,
                     Component.PRIMARY_INDEX,
                     Component.FILTER,
                     Component.COMPRESSION_INFO,
                     Component.STATS
                 }));

                 Iterator<Component> iter = components.iterator();
                 while (iter.hasNext()) {
                     Component component = iter.next();
                     if (!(new File(desc.filenameFor(component)).exists()))
                         iter.remove();
                 }
                 parsedFilenames.put(desc, components);
             }

             if (ksName == null || cfName == null)
             {
                 System.err.println("No valid sstables to split");
                 System.exit(1);
             }

             // Do not load sstables since they might be broken
             Keyspace keyspace = Keyspace.openWithoutSSTables(ksName);
             ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
             String snapshotName = "pre-split-" + System.currentTimeMillis();

             List<SSTableReader> sstables = new ArrayList<>();
             for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet())
             {
                 try
                 {
                     SSTableReader sstable = SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs);
                     if (!isSSTableLargerEnough(sstable, options.sizeInMB)) {
                         System.out.println(String.format("Skipping %s: it's size (%.3f MB) is less than the split size (%d MB)",
                                 sstable.getFilename(), ((sstable.onDiskLength() * 1.0d) / 1024L) / 1024L, options.sizeInMB));
                         continue;
                     }
                     sstables.add(sstable);

                     if (options.snapshot) {
                         File snapshotDirectory = Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
                         sstable.createLinks(snapshotDirectory.getPath());
                     }

                 }
                 catch (Exception e)
                 {
                     JVMStabilityInspector.inspectThrowable(e);
                     System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
                     if (options.debug)
                         e.printStackTrace(System.err);
                 }
             }
             if (sstables.isEmpty()) {
                 System.out.println("No sstables needed splitting.");
                 System.exit(0);
             }
             if (options.snapshot)
                 System.out.println(String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));

             for (SSTableReader sstable : sstables)
             {
                 try (LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.UNKNOWN, sstable))
                 {
                     new SSTableSplitter(cfs, transaction, options.sizeInMB).split();
                 }
                 catch (Exception e)
                 {
                     System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
                     if (options.debug)
                         e.printStackTrace(System.err);

                     sstable.selfRef().release();
                 }
             }
             CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
             LifecycleTransaction.waitForDeletions();
             System.exit(0); // We need that to stop non daemonized threads
         }
         catch (Exception e)
         {
             System.err.println(e.getMessage());
             if (options.debug)
                 e.printStackTrace(System.err);
             System.exit(1);
         }
     }

     /**
      * filter the sstable which size is less than the expected max sstable size.
      */
     private static boolean isSSTableLargerEnough(SSTableReader sstable, int sizeInMB) {
         return sstable.onDiskLength() > sizeInMB * 1024L * 1024L;
     }

     private static class Options
     {
         public final List<String> filenames;

         public boolean debug;
         public boolean snapshot;
         public int sizeInMB;

         private Options(List<String> filenames)
         {
             this.filenames = filenames;
         }

         public static Options parseArgs(String cmdArgs[])
         {
             CommandLineParser parser = new GnuParser();
             CmdLineOptions options = getCmdLineOptions();
             try
             {
                 CommandLine cmd = parser.parse(options, cmdArgs, false);

                 if (cmd.hasOption(HELP_OPTION))
                 {
                     printUsage(options);
                     System.exit(0);
                 }

                 String[] args = cmd.getArgs();
                 if (args.length == 0)
                 {
                     System.err.println("No sstables to split");
                     printUsage(options);
                     System.exit(1);
                 }
                 Options opts = new Options(Arrays.asList(args));
                 opts.debug = cmd.hasOption(DEBUG_OPTION);
                 opts.snapshot = !cmd.hasOption(NO_SNAPSHOT_OPTION);
                 opts.sizeInMB = DEFAULT_SSTABLE_SIZE;

                 if (cmd.hasOption(SIZE_OPTION))
                     opts.sizeInMB = Integer.parseInt(cmd.getOptionValue(SIZE_OPTION));

                 return opts;
             }
             catch (ParseException e)
             {
                 errorMsg(e.getMessage(), options);
                 return null;
             }
         }

         private static void errorMsg(String msg, CmdLineOptions options)
         {
             System.err.println(msg);
             printUsage(options);
             System.exit(1);
         }

         private static CmdLineOptions getCmdLineOptions()
         {
             CmdLineOptions options = new CmdLineOptions();
             options.addOption(null, DEBUG_OPTION,          "display stack traces");
             options.addOption("h",  HELP_OPTION,           "display this help message");
             options.addOption(null, NO_SNAPSHOT_OPTION,    "don't snapshot the sstables before splitting");
             options.addOption("s",  SIZE_OPTION, "size",   "maximum size in MB for the output sstables (default: " + DEFAULT_SSTABLE_SIZE + ")");
             return options;
         }

         public static void printUsage(CmdLineOptions options)
         {
             String usage = String.format("%s [options] <filename> [<filename>]*", TOOL_NAME);
             StringBuilder header = new StringBuilder();
             header.append("--\n");
             header.append("Split the provided sstables files in sstables of maximum provided file size (see option --" + SIZE_OPTION + ")." );
             header.append("\n--\n");
             header.append("Options are:");
             new HelpFormatter().printHelp(usage, header.toString(), options, "");
         }
     }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing,
	* software distributed under the License is distributed on an
	* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	* KIND, either express or implied. See the License for the
	* specific language governing permissions and limitations
	* under the License.
	*/
	package org.apache.cassandra.tools;

	import java.io.File;
	import java.util.*;
	import java.util.concurrent.TimeUnit;

	import org.apache.cassandra.config.Schema;
	import org.apache.cassandra.io.sstable.format.SSTableReader;
	import org.apache.commons.cli.*;

	import org.apache.cassandra.db.ColumnFamilyStore;
	import org.apache.cassandra.db.compaction.OperationType;
	import org.apache.cassandra.db.Directories;
	import org.apache.cassandra.db.Keyspace;
	import org.apache.cassandra.db.compaction.CompactionManager;
	import org.apache.cassandra.db.compaction.SSTableSplitter;
	import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
	import org.apache.cassandra.io.sstable.*;
	import org.apache.cassandra.utils.JVMStabilityInspector;
	import org.apache.cassandra.utils.Pair;

	import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;

	public class StandaloneSplitter
	{
	public static final int DEFAULT_SSTABLE_SIZE = 50;

	private static final String TOOL_NAME = "sstablessplit";
	private static final String DEBUG_OPTION = "debug";
	private static final String HELP_OPTION = "help";
	private static final String NO_SNAPSHOT_OPTION = "no-snapshot";
	private static final String SIZE_OPTION = "size";

	public static void main(String args[])
	{
	Options options = Options.parseArgs(args);
	Util.initDatabaseDescriptor();

	try
	{
	// load keyspace descriptions.
	Schema.instance.loadFromDisk(false);

	String ksName = null;
	String cfName = null;
	Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
	for (String filename : options.filenames)
	{
	File file = new File(filename);
	if (!file.exists()) {
	System.out.println("Skipping inexisting file " + file);
	continue;
	}

	Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
	if (pair == null) {
	System.out.println("Skipping non sstable file " + file);
	continue;
	}
	Descriptor desc = pair.left;

	if (ksName == null)
	ksName = desc.ksname;
	else if (!ksName.equals(desc.ksname))
	throw new IllegalArgumentException("All sstables must be part of the same keyspace");

	if (cfName == null)
	cfName = desc.cfname;
	else if (!cfName.equals(desc.cfname))
	throw new IllegalArgumentException("All sstables must be part of the same table");

	Set<Component> components = new HashSet<Component>(Arrays.asList(new Component[]{
	Component.DATA,
	Component.PRIMARY_INDEX,
	Component.FILTER,
	Component.COMPRESSION_INFO,
	Component.STATS
	}));

	Iterator<Component> iter = components.iterator();
	while (iter.hasNext()) {
	Component component = iter.next();
	if (!(new File(desc.filenameFor(component)).exists()))
	iter.remove();
	}
	parsedFilenames.put(desc, components);
	}

	if (ksName == null \|\| cfName == null)
	{
	System.err.println("No valid sstables to split");
	System.exit(1);
	}

	// Do not load sstables since they might be broken
	Keyspace keyspace = Keyspace.openWithoutSSTables(ksName);
	ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(cfName);
	String snapshotName = "pre-split-" + System.currentTimeMillis();

	List<SSTableReader> sstables = new ArrayList<>();
	for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet())
	{
	try
	{
	SSTableReader sstable = SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs);
	if (!isSSTableLargerEnough(sstable, options.sizeInMB)) {
	System.out.println(String.format("Skipping %s: it's size (%.3f MB) is less than the split size (%d MB)",
	sstable.getFilename(), ((sstable.onDiskLength() * 1.0d) / 1024L) / 1024L, options.sizeInMB));
	continue;
	}
	sstables.add(sstable);

	if (options.snapshot) {
	File snapshotDirectory = Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
	sstable.createLinks(snapshotDirectory.getPath());
	}

	}
	catch (Exception e)
	{
	JVMStabilityInspector.inspectThrowable(e);
	System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
	if (options.debug)
	e.printStackTrace(System.err);
	}
	}
	if (sstables.isEmpty()) {
	System.out.println("No sstables needed splitting.");
	System.exit(0);
	}
	if (options.snapshot)
	System.out.println(String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));

	for (SSTableReader sstable : sstables)
	{
	try (LifecycleTransaction transaction = LifecycleTransaction.offline(OperationType.UNKNOWN, sstable))
	{
	new SSTableSplitter(cfs, transaction, options.sizeInMB).split();
	}
	catch (Exception e)
	{
	System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
	if (options.debug)
	e.printStackTrace(System.err);

	sstable.selfRef().release();
	}
	}
	CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
	LifecycleTransaction.waitForDeletions();
	System.exit(0); // We need that to stop non daemonized threads
	}
	catch (Exception e)
	{
	System.err.println(e.getMessage());
	if (options.debug)
	e.printStackTrace(System.err);
	System.exit(1);
	}
	}

	/**
	* filter the sstable which size is less than the expected max sstable size.
	*/
	private static boolean isSSTableLargerEnough(SSTableReader sstable, int sizeInMB) {
	return sstable.onDiskLength() > sizeInMB * 1024L * 1024L;
	}

	private static class Options
	{
	public final List<String> filenames;

	public boolean debug;
	public boolean snapshot;
	public int sizeInMB;

	private Options(List<String> filenames)
	{
	this.filenames = filenames;
	}

	public static Options parseArgs(String cmdArgs[])
	{
	CommandLineParser parser = new GnuParser();
	CmdLineOptions options = getCmdLineOptions();
	try
	{
	CommandLine cmd = parser.parse(options, cmdArgs, false);

	if (cmd.hasOption(HELP_OPTION))
	{
	printUsage(options);
	System.exit(0);
	}

	String[] args = cmd.getArgs();
	if (args.length == 0)
	{
	System.err.println("No sstables to split");
	printUsage(options);
	System.exit(1);
	}
	Options opts = new Options(Arrays.asList(args));
	opts.debug = cmd.hasOption(DEBUG_OPTION);
	opts.snapshot = !cmd.hasOption(NO_SNAPSHOT_OPTION);
	opts.sizeInMB = DEFAULT_SSTABLE_SIZE;

	if (cmd.hasOption(SIZE_OPTION))
	opts.sizeInMB = Integer.parseInt(cmd.getOptionValue(SIZE_OPTION));

	return opts;
	}
	catch (ParseException e)
	{
	errorMsg(e.getMessage(), options);
	return null;
	}
	}

	private static void errorMsg(String msg, CmdLineOptions options)
	{
	System.err.println(msg);
	printUsage(options);
	System.exit(1);
	}

	private static CmdLineOptions getCmdLineOptions()
	{
	CmdLineOptions options = new CmdLineOptions();
	options.addOption(null, DEBUG_OPTION, "display stack traces");
	options.addOption("h", HELP_OPTION, "display this help message");
	options.addOption(null, NO_SNAPSHOT_OPTION, "don't snapshot the sstables before splitting");
	options.addOption("s", SIZE_OPTION, "size", "maximum size in MB for the output sstables (default: " + DEFAULT_SSTABLE_SIZE + ")");
	return options;
	}

	public static void printUsage(CmdLineOptions options)
	{
	String usage = String.format("%s [options] <filename> [<filename>]*", TOOL_NAME);
	StringBuilder header = new StringBuilder();
	header.append("--\n");
	header.append("Split the provided sstables files in sstables of maximum provided file size (see option --" + SIZE_OPTION + ")." );
	header.append("\n--\n");
	header.append("Options are:");
	new HelpFormatter().printHelp(usage, header.toString(), options, "");
	}
	}
	}