src/perftest/ExportStressTest.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.*;
import java.sql.*;
import java.util.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;

import org.apache.sqoop.Sqoop;
import org.apache.sqoop.SqoopOptions;
import org.apache.sqoop.tool.ExportTool;
import org.apache.sqoop.tool.SqoopTool;

/**
 * Stress test export procedure by running a large-scale export to MySQL.
 * This requires MySQL be configured with a database that can be accessed by
 * the specified username without a password. The user must be able to create
 * and drop tables in the database.
 *
 * Run with: src/scripts/run-perftest.sh ExportStressTest \
 *     (connect-str) (username)
 */
public class ExportStressTest extends Configured implements Tool {

  // Export 10 GB of data. Each record is ~100 bytes.
  public static final int NUM_FILES = 10;
  public static final int RECORDS_PER_FILE = 10 * 1024 * 1024;

  public static final String ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";

  public ExportStressTest() {
  }

  public void createFile(int fileId) throws IOException {
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    Path dirPath = new Path("ExportStressTest");
    fs.mkdirs(dirPath);
    Path filePath = new Path(dirPath, "input-" + fileId);

    OutputStream os = fs.create(filePath);
    Writer w = new BufferedWriter(new OutputStreamWriter(os));
    for (int i = 0; i < RECORDS_PER_FILE; i++) {
      long v = (long) i + ((long) RECORDS_PER_FILE * (long) fileId);
      w.write("" + v + "," + ALPHABET + ALPHABET + ALPHABET + ALPHABET + "\n");

    }
    w.close();
    os.close();
  }

  /** Create a set of data files to export. */
  public void createData() throws IOException {
    Configuration conf = getConf();
    FileSystem fs = FileSystem.get(conf);
    Path dirPath = new Path("ExportStressTest");
    if (fs.exists(dirPath)) {
      System.out.println(
          "Export directory appears to already exist. Skipping data-gen.");
      return;
    }

    for (int i = 0; i < NUM_FILES; i++) {
      createFile(i);
    }
  }

  /** Create a table to hold our results. Drop any existing definition. */
  public void createTable(String connectStr, String username) throws Exception {
    Class.forName("com.mysql.jdbc.Driver"); // Load mysql driver.

    Connection conn = DriverManager.getConnection(connectStr, username, null);
    conn.setAutoCommit(false);
    PreparedStatement stmt = conn.prepareStatement(
        "DROP TABLE IF EXISTS ExportStressTestTable",
        ResultSet.TYPE_FORWARD_ONLY,
        ResultSet.CONCUR_READ_ONLY);
    stmt.executeUpdate();
    stmt.close();

    stmt = conn.prepareStatement(
        "CREATE TABLE ExportStressTestTable(id INT NOT NULL PRIMARY KEY, "
        + "msg VARCHAR(110)) Engine=InnoDB", ResultSet.TYPE_FORWARD_ONLY,
        ResultSet.CONCUR_READ_ONLY);
    stmt.executeUpdate();
    stmt.close();
    conn.commit();
    conn.close();
  }

  /**
   * Actually run the export of the generated data to the user-created table.
   */
  public void runExport(String connectStr, String username) throws Exception {
    SqoopOptions options = new SqoopOptions(getConf());
    options.setConnectString(connectStr);
    options.setTableName("ExportStressTestTable");
    options.setUsername(username);
    options.setExportDir("ExportStressTest");
    options.setNumMappers(4);
    options.setLinesTerminatedBy('\n');
    options.setFieldsTerminatedBy(',');
    options.setExplicitOutputDelims(true);

    SqoopTool exportTool = new ExportTool();
    Sqoop sqoop = new Sqoop(exportTool, getConf(), options);
    int ret = Sqoop.runSqoop(sqoop, new String[0]);
    if (0 != ret) {
      throw new Exception("Error doing export; ret=" + ret);
    }
  }

  @Override
  public int run(String [] args) {
    String connectStr = args[0];
    String username = args[1];

    try {
      createData();
      createTable(connectStr, username);
      runExport(connectStr, username);
    } catch (Exception e) {
      System.err.println("Error: " + StringUtils.stringifyException(e));
      return 1;
    }

    return 0;
  }

  public static void main(String [] args) throws Exception {
    ExportStressTest test = new ExportStressTest();
    int ret = ToolRunner.run(test, args);
    System.exit(ret);
  }
}