/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hcatalog.cli;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
import org.apache.hcatalog.mapreduce.HCatBaseTest;
import org.apache.thrift.TException;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class TestSemanticAnalysis extends HCatBaseTest {

  private static final Logger LOG = LoggerFactory.getLogger(TestSemanticAnalysis.class);
  private static final String TBL_NAME = "junit_sem_analysis";

  private Driver hcatDriver = null;
  private String query;

  @Before
  public void setUpHCatDriver() throws IOException {
    if (hcatDriver == null) {
      HiveConf hcatConf = new HiveConf(hiveConf);
      hcatConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname,
          HCatSemanticAnalyzer.class.getName());
      hcatDriver = new Driver(hcatConf);
      SessionState.start(new CliSessionState(hcatConf));
    }
  }

  @Test
  public void testDescDB() throws CommandNeedRetryException, IOException {
    hcatDriver.run("drop database mydb cascade");
    assertEquals(0, hcatDriver.run("create database mydb").getResponseCode());
    CommandProcessorResponse resp = hcatDriver.run("describe database mydb");
    assertEquals(0, resp.getResponseCode());
    ArrayList<String> result = new ArrayList<String>();
    hcatDriver.getResults(result);
    assertTrue(result.get(0).contains("mydb.db"));
    hcatDriver.run("drop database mydb cascade");
  }

  @Test
  public void testCreateTblWithLowerCasePartNames() throws CommandNeedRetryException, MetaException, TException, NoSuchObjectException{
    driver.run("drop table junit_sem_analysis");
    CommandProcessorResponse resp = driver.run("create table junit_sem_analysis (a int) partitioned by (B string) stored as TEXTFILE");
    assertEquals(resp.getResponseCode(), 0);
    assertEquals(null, resp.getErrorMessage());
    Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
    assertEquals("Partition key name case problem", "b" , tbl.getPartitionKeys().get(0).getName());
    driver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testAlterTblFFpart() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException {

    driver.run("drop table junit_sem_analysis");
    driver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as TEXTFILE");
    driver.run("alter table junit_sem_analysis add partition (b='2010-10-10')");
    hcatDriver.run("alter table junit_sem_analysis partition (b='2010-10-10') set fileformat RCFILE");

    Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
    assertEquals(TextInputFormat.class.getName(), tbl.getSd().getInputFormat());
    assertEquals(HiveIgnoreKeyTextOutputFormat.class.getName(),tbl.getSd().getOutputFormat());

    List<String> partVals = new ArrayList<String>(1);
    partVals.add("2010-10-10");
    Partition part = client.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME, partVals);

    assertEquals(RCFileInputFormat.class.getName(),part.getSd().getInputFormat());
    assertEquals(RCFileOutputFormat.class.getName(),part.getSd().getOutputFormat());

    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testUsNonExistentDB() throws CommandNeedRetryException {
      CommandProcessorResponse resp = hcatDriver.run("use no_such_db");
      assertEquals(1, resp.getResponseCode());
  }

  @Test
  public void testDatabaseOperations() throws MetaException, CommandNeedRetryException {

    List<String> dbs = client.getAllDatabases();
    String testDb1 = "testdatabaseoperatons1";
    String testDb2 = "testdatabaseoperatons2";

    if (dbs.contains(testDb1.toLowerCase())){
      assertEquals(0,hcatDriver.run("drop database "+testDb1).getResponseCode());
    }

    if (dbs.contains(testDb2.toLowerCase())){
      assertEquals(0,hcatDriver.run("drop database "+testDb2).getResponseCode());
    }

    assertEquals(0,hcatDriver.run("create database "+testDb1).getResponseCode());
    assertTrue(client.getAllDatabases().contains(testDb1));
    assertEquals(0, hcatDriver.run("create database if not exists " + testDb1).getResponseCode());
    assertTrue(client.getAllDatabases().contains(testDb1));
    assertEquals(0,hcatDriver.run("create database if not exists "+testDb2).getResponseCode());
    assertTrue(client.getAllDatabases().contains(testDb2));

    assertEquals(0,hcatDriver.run("drop database "+testDb1).getResponseCode());
    assertEquals(0,hcatDriver.run("drop database "+testDb2).getResponseCode());
    assertFalse(client.getAllDatabases().contains(testDb1));
    assertFalse(client.getAllDatabases().contains(testDb2));
  }

  @Test
  public void testCreateTableIfNotExists() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException{

    hcatDriver.run("drop table "+ TBL_NAME);
    hcatDriver.run("create table junit_sem_analysis (a int) stored as RCFILE");
    Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
    List<FieldSchema> cols = tbl.getSd().getCols();
    assertEquals(1, cols.size());
    assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null)));
    assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat());
    assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat());

    CommandProcessorResponse resp = hcatDriver.run("create table if not exists junit_sem_analysis (a int) stored as RCFILE");
    assertEquals(0, resp.getResponseCode());
    assertNull(resp.getErrorMessage());
    tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
    cols = tbl.getSd().getCols();
    assertEquals(1, cols.size());
    assertTrue(cols.get(0).equals(new FieldSchema("a", "int",null)));
    assertEquals(RCFileInputFormat.class.getName(),tbl.getSd().getInputFormat());
    assertEquals(RCFileOutputFormat.class.getName(),tbl.getSd().getOutputFormat());

    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testAlterTblTouch() throws CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
    CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis touch");
    assertEquals(0, response.getResponseCode());

    hcatDriver.run("alter table junit_sem_analysis touch partition (b='12')");
    assertEquals(0, response.getResponseCode());

    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testChangeColumns() throws CommandNeedRetryException{
    hcatDriver.run("drop table junit_sem_analysis");
    hcatDriver.run("create table junit_sem_analysis (a int, c string) partitioned by (b string) stored as RCFILE");
    CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis change a a1 int");
    assertEquals(0, response.getResponseCode());

    response = hcatDriver.run("alter table junit_sem_analysis change a1 a string");
    assertEquals(0, response.getResponseCode());

    response = hcatDriver.run("alter table junit_sem_analysis change a a int after c");
    assertEquals(0, response.getResponseCode());
    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testAddReplaceCols() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    hcatDriver.run("create table junit_sem_analysis (a int, c string) partitioned by (b string) stored as RCFILE");
    CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis replace columns (a1 tinyint)");
    assertEquals(0, response.getResponseCode());

    response = hcatDriver.run("alter table junit_sem_analysis add columns (d tinyint)");
    assertEquals(0, response.getResponseCode());
    assertNull(response.getErrorMessage());

    response = hcatDriver.run("describe extended junit_sem_analysis");
    assertEquals(0, response.getResponseCode());
    Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
    List<FieldSchema> cols = tbl.getSd().getCols();
    assertEquals(2, cols.size());
    assertTrue(cols.get(0).equals(new FieldSchema("a1", "tinyint", null)));
    assertTrue(cols.get(1).equals(new FieldSchema("d", "tinyint", null)));
    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testAlterTblClusteredBy() throws CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
    CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis clustered by (a) into 7 buckets");
    assertEquals(0, response.getResponseCode());
    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testAlterTableSetFF() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");

    Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
    assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat());
    assertEquals(RCFileOutputFormat.class.getName(),tbl.getSd().getOutputFormat());

    hcatDriver.run("alter table junit_sem_analysis set fileformat INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " +
        "'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver'");
    hcatDriver.run("desc extended junit_sem_analysis");

    tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
    assertEquals(RCFileInputFormat.class.getName(),tbl.getSd().getInputFormat());
    assertEquals(RCFileOutputFormat.class.getName(),tbl.getSd().getOutputFormat());

    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testAddPartFail() throws CommandNeedRetryException{

    driver.run("drop table junit_sem_analysis");
    driver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
    CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis add partition (b='2') location 'README.txt'");
    assertEquals(0, response.getResponseCode());
    driver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testAddPartPass() throws IOException, CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
    CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis add partition (b='2') location '" + TEST_DATA_DIR + "'");
    assertEquals(0, response.getResponseCode());
    assertNull(response.getErrorMessage());
    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testCTAS() throws CommandNeedRetryException{
    hcatDriver.run("drop table junit_sem_analysis");
    query = "create table junit_sem_analysis (a int) as select * from tbl2";
    CommandProcessorResponse response = hcatDriver.run(query);
    assertEquals(40000, response.getResponseCode());
    assertTrue(response.getErrorMessage().contains("FAILED: SemanticException Operation not supported. Create table as Select is not a valid operation."));
    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testStoredAs() throws CommandNeedRetryException{
    hcatDriver.run("drop table junit_sem_analysis");
    query = "create table junit_sem_analysis (a int)";
    CommandProcessorResponse response = hcatDriver.run(query);
    assertEquals(0, response.getResponseCode());
    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testAddDriverInfo() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    query =  "create table junit_sem_analysis (a int) partitioned by (b string)  stored as " +
        "INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " +
        "'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver' ";
    assertEquals(0,hcatDriver.run(query).getResponseCode());

    Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME);
    assertEquals(RCFileInputFormat.class.getName(),tbl.getSd().getInputFormat());
    assertEquals(RCFileOutputFormat.class.getName(),tbl.getSd().getOutputFormat());

    hcatDriver.run("drop table junit_sem_analysis");
  }

  @Test
  public void testInvalidateNonStringPartition() throws IOException, CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    query =  "create table junit_sem_analysis (a int) partitioned by (b int)  stored as RCFILE";

    CommandProcessorResponse response = hcatDriver.run(query);
    assertEquals(40000,response.getResponseCode());
    assertEquals("FAILED: SemanticException Operation not supported. HCatalog only supports partition columns of type string. For column: b Found type: int",
        response.getErrorMessage());

  }

  @Test
  public void testInvalidateSeqFileStoredAs() throws IOException, CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    query =  "create table junit_sem_analysis (a int) partitioned by (b string)  stored as SEQUENCEFILE";

    CommandProcessorResponse response = hcatDriver.run(query);
    assertEquals(0,response.getResponseCode());

  }

  @Test
  public void testInvalidateTextFileStoredAs() throws IOException, CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    query =  "create table junit_sem_analysis (a int) partitioned by (b string)  stored as TEXTFILE";

    CommandProcessorResponse response = hcatDriver.run(query);
    assertEquals(0,response.getResponseCode());

  }

  @Test
  public void testInvalidateClusteredBy() throws IOException, CommandNeedRetryException{

    hcatDriver.run("drop table junit_sem_analysis");
    query =  "create table junit_sem_analysis (a int) partitioned by (b string) clustered by (a) into 10 buckets stored as TEXTFILE";

    CommandProcessorResponse response = hcatDriver.run(query);
    assertEquals(0,response.getResponseCode());
  }

  @Test
  public void testCTLFail() throws IOException, CommandNeedRetryException{

    driver.run("drop table junit_sem_analysis");
    driver.run("drop table like_table");
    query =  "create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE";

    driver.run(query);
    query = "create table like_table like junit_sem_analysis";
    CommandProcessorResponse response = hcatDriver.run(query);
    assertEquals(0,response.getResponseCode());
  }

  @Test
  public void testCTLPass() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException{

    try{
      hcatDriver.run("drop table junit_sem_analysis");
    }
    catch( Exception e){
      LOG.error("Error in drop table.",e);
    }
    query =  "create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE";

    hcatDriver.run(query);
    String likeTbl = "like_table";
    hcatDriver.run("drop table "+likeTbl);
    query = "create table like_table like junit_sem_analysis";
    CommandProcessorResponse resp = hcatDriver.run(query);
    assertEquals(0, resp.getResponseCode());
//    Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, likeTbl);
//    assertEquals(likeTbl,tbl.getTableName());
//    List<FieldSchema> cols = tbl.getSd().getCols();
//    assertEquals(1, cols.size());
//    assertEquals(new FieldSchema("a", "int", null), cols.get(0));
//    assertEquals("org.apache.hadoop.hive.ql.io.RCFileInputFormat",tbl.getSd().getInputFormat());
//    assertEquals("org.apache.hadoop.hive.ql.io.RCFileOutputFormat",tbl.getSd().getOutputFormat());
//    Map<String, String> tblParams = tbl.getParameters();
//    assertEquals("org.apache.hadoop.hive.hcat.rcfile.RCFileInputStorageDriver", tblParams.get("hcat.isd"));
//    assertEquals("org.apache.hadoop.hive.hcat.rcfile.RCFileOutputStorageDriver", tblParams.get("hcat.osd"));
//
//    hcatDriver.run("drop table junit_sem_analysis");
//    hcatDriver.run("drop table "+likeTbl);
  }

// This test case currently fails, since add partitions don't inherit anything from tables.

//  public void testAddPartInheritDrivers() throws MetaException, TException, NoSuchObjectException{
//
//    hcatDriver.run("drop table "+TBL_NAME);
//    hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE");
//    hcatDriver.run("alter table "+TBL_NAME+" add partition (b='2010-10-10')");
//
//    List<String> partVals = new ArrayList<String>(1);
//    partVals.add("2010-10-10");
//
//    Map<String,String> map = client.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME, partVals).getParameters();
//    assertEquals(map.get(InitializeInput.HOWL_ISD_CLASS), RCFileInputStorageDriver.class.getName());
//    assertEquals(map.get(InitializeInput.HOWL_OSD_CLASS), RCFileOutputStorageDriver.class.getName());
//  }
}