src/test/org/apache/sqoop/util/TestOptionsFileExpansion.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.sqoop.util;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;

import org.apache.sqoop.util.OptionsFileUtil;
import org.junit.Assert;

import org.apache.sqoop.Sqoop;
import org.junit.Test;

/**
 * Tests various options file loading scenarios.
 */
public class TestOptionsFileExpansion {

  /**
   * Text from options file 1. Each string represents a new line.
   */
  private static final String[] OPTIONS_FILE_TEXT1 = new String[] {
    "--foo",
    "-bar",
    "--",
    "--XYZ",
  };

  /**
   * Expected options parsed out from options file 1.
   */
  private static final String[] OPTIONS_FILE_TEXT1_OUTPUT = new String[] {
    "--foo",
    "-bar",
    "--",
    "--XYZ",
  };

  /**
   * Text for options file 2. Each string represents a new line. This
   * contains empty lines, comments and optinos that extend to multiple lines.
   */
  private static final String[] OPTIONS_FILE_TEXT2 = new String[] {
    "--archives",
    "tools.jar,archive.jar,test.jar,\\",
    "ldap.jar,sasl.jar",
    "--connect",
    "jdbc:jdbcspy:localhost:1521:test",
    "--username",
    "superman",
    "--password",
    "",
    "# Ironic password.",
    "# No one will ever guess.",
    "kryptonite",
  };

  /**
   * Expected options parsed out from file 2.
   */
  private static final String[] OPTIONS_FILE_TEXT2_OUTPUT = new String[] {
    "--archives",
    "tools.jar,archive.jar,test.jar,ldap.jar,sasl.jar",
    "--connect",
    "jdbc:jdbcspy:localhost:1521:test",
    "--username",
    "superman",
    "--password",
    "kryptonite",
  };

  /**
   * Text for options file 4. This contains options that represent empty
   * strings or strings that have leading and trailing spaces.
   */
  private static final String[] OPTIONS_FILE_TEXT3 = new String[] {
    "-",
    "\"     leading spaces\"",
    "'        leading and trailing spaces    '",
    "\"\"",
    "''",
  };

  /**
   * Expected options parsed out from file 3.
   */
  private static final String[] OPTIONS_FILE_TEXT3_OUTPUT = new String[] {
    "-",
    "     leading spaces",
    "        leading and trailing spaces    ",
    "",
    "",
  };

  /**
   * Text for options file 4. This file has an invalid entry in the last line
   * which will cause it to fail to load.
   */
  private static final String[] OPTIONS_FILE_TEXT4 = new String[] {
    "--abcd",
    "--efgh",
    "# foo",
    "# bar",
    "XYZ\\",
  };

  /**
   * Text for options file 5. This file has an invalid entry in the second line
   * where there is a starting single quote character that is not terminating.
   */
  private static final String[] OPTIONS_FILE_TEXT5 = new String[] {
    "-abcd",
    "\'",
    "--foo",
  };

  /**
   * Text for options file 6. This file has an invalid entry in the second line
   * where a quoted string extends into the following line.
   */
  private static final String[] OPTIONS_FILE_TEXT6 = new String[] {
    "--abcd",
    "' the quick brown fox \\",
    "jumped over the lazy dog'",
    "--efgh",
  };

  @Test
  public void testOptionsFiles() throws Exception {
    checkOptionsFile(OPTIONS_FILE_TEXT1, OPTIONS_FILE_TEXT1_OUTPUT);
    checkOptionsFile(OPTIONS_FILE_TEXT2, OPTIONS_FILE_TEXT2_OUTPUT);
    checkOptionsFile(OPTIONS_FILE_TEXT3, OPTIONS_FILE_TEXT3_OUTPUT);
  }

  @Test
  public void testInvalidOptionsFile() {
    checkInvalidOptionsFile(OPTIONS_FILE_TEXT4);
    checkInvalidOptionsFile(OPTIONS_FILE_TEXT5);
  }

  @Test
  public void testMultilineQuotedText() {
    try {
      checkOptionsFile(OPTIONS_FILE_TEXT6, new String[] {});
      Assert.assertTrue(false);
    } catch (Exception ex) {
      Assert.assertTrue(
          ex.getMessage().startsWith("Multiline quoted strings not supported"));
    }
  }

  @Test
  public void testValidFreeFormQueryNoQuotes() throws Exception {
    String[] input = new String[]{
        "--query",
        "SELECT * FROM table",
    };

    String[] output = new String[] {
        "--query",
        "SELECT * FROM table",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testValidFreeFormQuerySingleQuotesStartAndEnd() throws Exception {
    String[] input = new String[]{
        "--query",
        "'SELECT * FROM table'",
    };

    String[] output = new String[]{
        "--query",
        "SELECT * FROM table",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testValidFreeFormQueryDoubleQuotesStartAndEnd() throws Exception {
    String[] input = new String[]{
        "--query",
        "\"SELECT * FROM table\"",
    };

    String[] output = new String[]{
        "--query",
        "SELECT * FROM table",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testValidFreeFormQuerySingleQuotesInWhere() throws Exception {
    String[] input = new String[]{
        "--query",
        "SELECT * FROM table WHERE a = '1'",
    };

    String[] output = new String[]{
        "--query",
        "SELECT * FROM table WHERE a = '1'",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testValidFreeFormQuerySingleAndDoubleQuotesInWhere() throws Exception {
    String[] input = new String[] {
        "--query",
        "SELECT * FROM table WHERE a = '1' AND b = \"testing\"",
    };

    String[] output = new String[] {
        "--query",
        "SELECT * FROM table WHERE a = '1' AND b = \"testing\"",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testValidFreeFormQueryQuotesInTableNameAndColumnName() throws Exception {
    String[] input = new String[] {
        "--query",
        "select * from `test\"test`  where `c'c`  = 'a'",
    };

    String[] output = new String[] {
        "--query",
        "select * from `test\"test`  where `c'c`  = 'a'",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testValidFreeFormQueryQuotesInTableNameAndColumnName2() throws Exception {
    String[] input = new String[] {
        "--query",
        "select * from `test\"test`  where `c'c`  = 'a\"'",
    };

    String[] output = new String[] {
        "--query",
        "select * from `test\"test`  where `c'c`  = 'a\"'",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testValidFreeFormQueryQuotesInTableNameAndColumnName3() throws Exception {
    String[] input = new String[] {
        "--query",
        "select * from `test\"test`  where `c'c`  = \"\"",
    };

    String[] output = new String[] {
        "--query",
        "select * from `test\"test`  where `c'c`  = \"\"",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testValidFreeFormQueryQuotesInTableNameAndColumnName4() throws Exception {
    String[] input = new String[] {
        "--query",
        "select * from test  where a  = \"\\\"\"",
    };

    String[] output = new String[] {
        "--query",
        "select * from test  where a  = \"\\\"\"",
    };

    checkOptionsFile(input, output);
  }

  @Test
  public void testInvalidFreeFormQueryEndingSingleQuoteOnly() throws Exception {
    String[] input = new String[]{
        "--query",
        "SELECT * FROM table'",
    };

    checkInvalidOptionsFile(input);
  }

  @Test
  public void testInvalidFreeFormQuerySingleQuoteStartDoubleQuoteEnd() throws Exception {

    String[] input = new String[]{
        "--query",
        "'SELECT * FROM table\"",
    };

    checkInvalidOptionsFile(input);
  }

  private void checkInvalidOptionsFile(String[] fileContents) {
    try {
      checkOptionsFile(fileContents, new String[] {});
      Assert.assertTrue(false);
    } catch (Exception ex) {
      Assert.assertTrue(ex.getMessage().startsWith("Malformed option"));
    }
  }

  private void checkOptionsFile(String[] fileContent, String[] expectedOptions)
    throws Exception {
    String[] prefix0 = new String[] { };
    String[] suffix0 = new String[] { };

    checkOutput(prefix0, suffix0, fileContent, expectedOptions);

    String[] prefix1 = new String[] { "--nomnom" };
    String[] suffix1 = new String[] { };

    checkOutput(prefix1, suffix1,
        fileContent, expectedOptions);

    String[] prefix2 = new String[] { };
    String[] suffix2 = new String[] { "yIkes" };

    checkOutput(prefix2, suffix2,
        fileContent, expectedOptions);

    String[] prefix3 = new String[] { "foo", "bar" };
    String[] suffix3 = new String[] { "xyz", "abc" };

    checkOutput(prefix3, suffix3,
        fileContent, expectedOptions);
  }


  /**
   * Uses the given prefix and suffix to create the original args array which
   * contains two entries between the prefix and suffix entries that specify
   * the options file. The options file is dynamically created using the
   * contents of the third array - fileContent. Once this is expanded, the
   * expanded arguments are compared to see if they are same as prefix entries
   * followed by parsed arguments from the options file, followed by suffix
   * entries.
   * @param prefix
   * @param suffix
   * @param fileContent
   * @param expectedContent
   * @throws Exception
   */
  private void checkOutput(String[] prefix, String[] suffix,
      String[] fileContent, String[] expectedContent) throws Exception {

    String[] args = new String[prefix.length + 2 + suffix.length];

    for (int i = 0; i < prefix.length; i++) {
      args[i] = prefix[i];
    }
    args[prefix.length] = Sqoop.SQOOP_OPTIONS_FILE_SPECIFIER;
    args[prefix.length + 1] =  createOptionsFile(fileContent);
    for (int j = 0; j < suffix.length; j++) {
      args[j + 2 + prefix.length] = suffix[j];
    }

    String[] expandedArgs = OptionsFileUtil.expandArguments(args);

    assertSame(prefix, expectedContent, suffix, expandedArgs);

  }

  private void assertSame(String[] prefix, String[] content, String[] suffix,
      String[] actual) {
    Assert.assertTrue(prefix.length + content.length + suffix.length
        == actual.length);

    for (int i = 0; i < prefix.length; i++) {
      Assert.assertTrue(actual[i].equals(prefix[i]));
    }

    for (int i = 0; i < content.length; i++) {
      Assert.assertTrue(actual[i + prefix.length].equals(content[i]));
    }

    for (int i = 0; i < suffix.length; i++) {
      Assert.assertTrue(actual[i + prefix.length + content.length].equals(
          suffix[i]));
    }
  }

  private String createOptionsFile(String[] data) throws Exception {
    File file = File.createTempFile("options", ".opf");
    file.deleteOnExit();

    BufferedWriter writer = null;
    try {
      writer = new BufferedWriter(new FileWriter(file));
      for (String datum : data) {
        writer.write(datum);
        writer.newLine();
      }
    } finally {
      if (writer != null) {
        try {
          writer.close();
        } catch (IOException ex) {
          // No handling required
        }
      }
    }

    return file.getAbsolutePath();
  }
}