| /** |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, |
| * software distributed under the License is distributed on an |
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| * KIND, either express or implied. See the License for the |
| * specific language governing permissions and limitations |
| * under the License. |
| */ |
| package org.apache.hcatalog.cli; |
| |
| import static org.junit.Assert.assertEquals; |
| import static org.junit.Assert.assertFalse; |
| import static org.junit.Assert.assertNull; |
| import static org.junit.Assert.assertTrue; |
| |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.List; |
| |
| import org.apache.hadoop.hive.cli.CliSessionState; |
| import org.apache.hadoop.hive.conf.HiveConf; |
| import org.apache.hadoop.hive.metastore.MetaStoreUtils; |
| import org.apache.hadoop.hive.metastore.api.FieldSchema; |
| import org.apache.hadoop.hive.metastore.api.MetaException; |
| import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; |
| import org.apache.hadoop.hive.metastore.api.Partition; |
| import org.apache.hadoop.hive.metastore.api.Table; |
| import org.apache.hadoop.hive.ql.CommandNeedRetryException; |
| import org.apache.hadoop.hive.ql.Driver; |
| import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; |
| import org.apache.hadoop.hive.ql.io.RCFileInputFormat; |
| import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; |
| import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; |
| import org.apache.hadoop.hive.ql.session.SessionState; |
| import org.apache.hadoop.mapred.TextInputFormat; |
| import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; |
| import org.apache.hcatalog.mapreduce.HCatBaseTest; |
| import org.apache.thrift.TException; |
| import org.junit.Before; |
| import org.junit.Test; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| |
| public class TestSemanticAnalysis extends HCatBaseTest { |
| |
| private static final Logger LOG = LoggerFactory.getLogger(TestSemanticAnalysis.class); |
| private static final String TBL_NAME = "junit_sem_analysis"; |
| |
| private Driver hcatDriver = null; |
| private String query; |
| |
| @Before |
| public void setUpHCatDriver() throws IOException { |
| if (hcatDriver == null) { |
| HiveConf hcatConf = new HiveConf(hiveConf); |
| hcatConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, |
| HCatSemanticAnalyzer.class.getName()); |
| hcatDriver = new Driver(hcatConf); |
| SessionState.start(new CliSessionState(hcatConf)); |
| } |
| } |
| |
| @Test |
| public void testDescDB() throws CommandNeedRetryException, IOException { |
| hcatDriver.run("drop database mydb cascade"); |
| assertEquals(0, hcatDriver.run("create database mydb").getResponseCode()); |
| CommandProcessorResponse resp = hcatDriver.run("describe database mydb"); |
| assertEquals(0, resp.getResponseCode()); |
| ArrayList<String> result = new ArrayList<String>(); |
| hcatDriver.getResults(result); |
| assertTrue(result.get(0).contains("mydb.db")); |
| hcatDriver.run("drop database mydb cascade"); |
| } |
| |
| @Test |
| public void testCreateTblWithLowerCasePartNames() throws CommandNeedRetryException, MetaException, TException, NoSuchObjectException { |
| driver.run("drop table junit_sem_analysis"); |
| CommandProcessorResponse resp = driver.run("create table junit_sem_analysis (a int) partitioned by (B string) stored as TEXTFILE"); |
| assertEquals(resp.getResponseCode(), 0); |
| assertEquals(null, resp.getErrorMessage()); |
| Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); |
| assertEquals("Partition key name case problem", "b", tbl.getPartitionKeys().get(0).getName()); |
| driver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testAlterTblFFpart() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException { |
| |
| driver.run("drop table junit_sem_analysis"); |
| driver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as TEXTFILE"); |
| driver.run("alter table junit_sem_analysis add partition (b='2010-10-10')"); |
| hcatDriver.run("alter table junit_sem_analysis partition (b='2010-10-10') set fileformat RCFILE"); |
| |
| Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); |
| assertEquals(TextInputFormat.class.getName(), tbl.getSd().getInputFormat()); |
| assertEquals(HiveIgnoreKeyTextOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); |
| |
| List<String> partVals = new ArrayList<String>(1); |
| partVals.add("2010-10-10"); |
| Partition part = client.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME, partVals); |
| |
| assertEquals(RCFileInputFormat.class.getName(), part.getSd().getInputFormat()); |
| assertEquals(RCFileOutputFormat.class.getName(), part.getSd().getOutputFormat()); |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testUsNonExistentDB() throws CommandNeedRetryException { |
| CommandProcessorResponse resp = hcatDriver.run("use no_such_db"); |
| assertEquals(1, resp.getResponseCode()); |
| } |
| |
| @Test |
| public void testDatabaseOperations() throws MetaException, CommandNeedRetryException { |
| |
| List<String> dbs = client.getAllDatabases(); |
| String testDb1 = "testdatabaseoperatons1"; |
| String testDb2 = "testdatabaseoperatons2"; |
| |
| if (dbs.contains(testDb1.toLowerCase())) { |
| assertEquals(0, hcatDriver.run("drop database " + testDb1).getResponseCode()); |
| } |
| |
| if (dbs.contains(testDb2.toLowerCase())) { |
| assertEquals(0, hcatDriver.run("drop database " + testDb2).getResponseCode()); |
| } |
| |
| assertEquals(0, hcatDriver.run("create database " + testDb1).getResponseCode()); |
| assertTrue(client.getAllDatabases().contains(testDb1)); |
| assertEquals(0, hcatDriver.run("create database if not exists " + testDb1).getResponseCode()); |
| assertTrue(client.getAllDatabases().contains(testDb1)); |
| assertEquals(0, hcatDriver.run("create database if not exists " + testDb2).getResponseCode()); |
| assertTrue(client.getAllDatabases().contains(testDb2)); |
| |
| assertEquals(0, hcatDriver.run("drop database " + testDb1).getResponseCode()); |
| assertEquals(0, hcatDriver.run("drop database " + testDb2).getResponseCode()); |
| assertFalse(client.getAllDatabases().contains(testDb1)); |
| assertFalse(client.getAllDatabases().contains(testDb2)); |
| } |
| |
| @Test |
| public void testCreateTableIfNotExists() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table " + TBL_NAME); |
| hcatDriver.run("create table junit_sem_analysis (a int) stored as RCFILE"); |
| Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); |
| List<FieldSchema> cols = tbl.getSd().getCols(); |
| assertEquals(1, cols.size()); |
| assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null))); |
| assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); |
| assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); |
| |
| CommandProcessorResponse resp = hcatDriver.run("create table if not exists junit_sem_analysis (a int) stored as RCFILE"); |
| assertEquals(0, resp.getResponseCode()); |
| assertNull(resp.getErrorMessage()); |
| tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); |
| cols = tbl.getSd().getCols(); |
| assertEquals(1, cols.size()); |
| assertTrue(cols.get(0).equals(new FieldSchema("a", "int", null))); |
| assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); |
| assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testAlterTblTouch() throws CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"); |
| CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis touch"); |
| assertEquals(0, response.getResponseCode()); |
| |
| hcatDriver.run("alter table junit_sem_analysis touch partition (b='12')"); |
| assertEquals(0, response.getResponseCode()); |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testChangeColumns() throws CommandNeedRetryException { |
| hcatDriver.run("drop table junit_sem_analysis"); |
| hcatDriver.run("create table junit_sem_analysis (a int, c string) partitioned by (b string) stored as RCFILE"); |
| CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis change a a1 int"); |
| assertEquals(0, response.getResponseCode()); |
| |
| response = hcatDriver.run("alter table junit_sem_analysis change a1 a string"); |
| assertEquals(0, response.getResponseCode()); |
| |
| response = hcatDriver.run("alter table junit_sem_analysis change a a int after c"); |
| assertEquals(0, response.getResponseCode()); |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testAddReplaceCols() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| hcatDriver.run("create table junit_sem_analysis (a int, c string) partitioned by (b string) stored as RCFILE"); |
| CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis replace columns (a1 tinyint)"); |
| assertEquals(0, response.getResponseCode()); |
| |
| response = hcatDriver.run("alter table junit_sem_analysis add columns (d tinyint)"); |
| assertEquals(0, response.getResponseCode()); |
| assertNull(response.getErrorMessage()); |
| |
| response = hcatDriver.run("describe extended junit_sem_analysis"); |
| assertEquals(0, response.getResponseCode()); |
| Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); |
| List<FieldSchema> cols = tbl.getSd().getCols(); |
| assertEquals(2, cols.size()); |
| assertTrue(cols.get(0).equals(new FieldSchema("a1", "tinyint", null))); |
| assertTrue(cols.get(1).equals(new FieldSchema("d", "tinyint", null))); |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testAlterTblClusteredBy() throws CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"); |
| CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis clustered by (a) into 7 buckets"); |
| assertEquals(0, response.getResponseCode()); |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testAlterTableSetFF() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"); |
| |
| Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); |
| assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); |
| assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); |
| |
| hcatDriver.run("alter table junit_sem_analysis set fileformat INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " + |
| "'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver'"); |
| hcatDriver.run("desc extended junit_sem_analysis"); |
| |
| tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); |
| assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); |
| assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testAddPartFail() throws CommandNeedRetryException { |
| |
| driver.run("drop table junit_sem_analysis"); |
| driver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"); |
| CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis add partition (b='2') location 'README.txt'"); |
| assertEquals(0, response.getResponseCode()); |
| driver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testAddPartPass() throws IOException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"); |
| CommandProcessorResponse response = hcatDriver.run("alter table junit_sem_analysis add partition (b='2') location '" + TEST_DATA_DIR + "'"); |
| assertEquals(0, response.getResponseCode()); |
| assertNull(response.getErrorMessage()); |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testCTAS() throws CommandNeedRetryException { |
| hcatDriver.run("drop table junit_sem_analysis"); |
| query = "create table junit_sem_analysis (a int) as select * from tbl2"; |
| CommandProcessorResponse response = hcatDriver.run(query); |
| assertEquals(40000, response.getResponseCode()); |
| assertTrue(response.getErrorMessage().contains("FAILED: SemanticException Operation not supported. Create table as Select is not a valid operation.")); |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testStoredAs() throws CommandNeedRetryException { |
| hcatDriver.run("drop table junit_sem_analysis"); |
| query = "create table junit_sem_analysis (a int)"; |
| CommandProcessorResponse response = hcatDriver.run(query); |
| assertEquals(0, response.getResponseCode()); |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testAddDriverInfo() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as " + |
| "INPUTFORMAT 'org.apache.hadoop.hive.ql.io.RCFileInputFormat' OUTPUTFORMAT " + |
| "'org.apache.hadoop.hive.ql.io.RCFileOutputFormat' inputdriver 'mydriver' outputdriver 'yourdriver' "; |
| assertEquals(0, hcatDriver.run(query).getResponseCode()); |
| |
| Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); |
| assertEquals(RCFileInputFormat.class.getName(), tbl.getSd().getInputFormat()); |
| assertEquals(RCFileOutputFormat.class.getName(), tbl.getSd().getOutputFormat()); |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } |
| |
| @Test |
| public void testInvalidateNonStringPartition() throws IOException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| query = "create table junit_sem_analysis (a int) partitioned by (b int) stored as RCFILE"; |
| |
| CommandProcessorResponse response = hcatDriver.run(query); |
| assertEquals(40000, response.getResponseCode()); |
| assertEquals("FAILED: SemanticException Operation not supported. HCatalog only supports partition columns of type string. For column: b Found type: int", |
| response.getErrorMessage()); |
| |
| } |
| |
| @Test |
| public void testInvalidateSeqFileStoredAs() throws IOException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as SEQUENCEFILE"; |
| |
| CommandProcessorResponse response = hcatDriver.run(query); |
| assertEquals(0, response.getResponseCode()); |
| |
| } |
| |
| @Test |
| public void testInvalidateTextFileStoredAs() throws IOException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as TEXTFILE"; |
| |
| CommandProcessorResponse response = hcatDriver.run(query); |
| assertEquals(0, response.getResponseCode()); |
| |
| } |
| |
| @Test |
| public void testInvalidateClusteredBy() throws IOException, CommandNeedRetryException { |
| |
| hcatDriver.run("drop table junit_sem_analysis"); |
| query = "create table junit_sem_analysis (a int) partitioned by (b string) clustered by (a) into 10 buckets stored as TEXTFILE"; |
| |
| CommandProcessorResponse response = hcatDriver.run(query); |
| assertEquals(0, response.getResponseCode()); |
| } |
| |
| @Test |
| public void testCTLFail() throws IOException, CommandNeedRetryException { |
| |
| driver.run("drop table junit_sem_analysis"); |
| driver.run("drop table like_table"); |
| query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"; |
| |
| driver.run(query); |
| query = "create table like_table like junit_sem_analysis"; |
| CommandProcessorResponse response = hcatDriver.run(query); |
| assertEquals(0, response.getResponseCode()); |
| } |
| |
| @Test |
| public void testCTLPass() throws IOException, MetaException, TException, NoSuchObjectException, CommandNeedRetryException { |
| |
| try { |
| hcatDriver.run("drop table junit_sem_analysis"); |
| } catch (Exception e) { |
| LOG.error("Error in drop table.", e); |
| } |
| query = "create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"; |
| |
| hcatDriver.run(query); |
| String likeTbl = "like_table"; |
| hcatDriver.run("drop table " + likeTbl); |
| query = "create table like_table like junit_sem_analysis"; |
| CommandProcessorResponse resp = hcatDriver.run(query); |
| assertEquals(0, resp.getResponseCode()); |
| // Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, likeTbl); |
| // assertEquals(likeTbl,tbl.getTableName()); |
| // List<FieldSchema> cols = tbl.getSd().getCols(); |
| // assertEquals(1, cols.size()); |
| // assertEquals(new FieldSchema("a", "int", null), cols.get(0)); |
| // assertEquals("org.apache.hadoop.hive.ql.io.RCFileInputFormat",tbl.getSd().getInputFormat()); |
| // assertEquals("org.apache.hadoop.hive.ql.io.RCFileOutputFormat",tbl.getSd().getOutputFormat()); |
| // Map<String, String> tblParams = tbl.getParameters(); |
| // assertEquals("org.apache.hadoop.hive.hcat.rcfile.RCFileInputStorageDriver", tblParams.get("hcat.isd")); |
| // assertEquals("org.apache.hadoop.hive.hcat.rcfile.RCFileOutputStorageDriver", tblParams.get("hcat.osd")); |
| // |
| // hcatDriver.run("drop table junit_sem_analysis"); |
| // hcatDriver.run("drop table "+likeTbl); |
| } |
| |
| // This test case currently fails, since add partitions don't inherit anything from tables. |
| |
| // public void testAddPartInheritDrivers() throws MetaException, TException, NoSuchObjectException{ |
| // |
| // hcatDriver.run("drop table "+TBL_NAME); |
| // hcatDriver.run("create table junit_sem_analysis (a int) partitioned by (b string) stored as RCFILE"); |
| // hcatDriver.run("alter table "+TBL_NAME+" add partition (b='2010-10-10')"); |
| // |
| // List<String> partVals = new ArrayList<String>(1); |
| // partVals.add("2010-10-10"); |
| // |
| // Map<String,String> map = client.getPartition(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME, partVals).getParameters(); |
| // assertEquals(map.get(InitializeInput.HOWL_ISD_CLASS), RCFileInputStorageDriver.class.getName()); |
| // assertEquals(map.get(InitializeInput.HOWL_OSD_CLASS), RCFileOutputStorageDriver.class.getName()); |
| // } |
| } |