blob: 5afebaffaf22ce0dec2cf543d7379fe5cb091091 [file] [log] [blame]
#!/bin/env python
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""
/***************************************************************************
*
* @file test_sys_alter_schema_change_modify.py
* @date 2020-02-04
* @brief This file is a test file for Palo bitmap index.
*
**************************************************************************/
"""
import sys
import random
import time
sys.path.append("../")
from data import bitmap_index as DATA
from lib import palo_config
from lib import palo_client
from lib import util
config = palo_config.config
LOG = palo_client.LOG
L = palo_client.L
broker_info = palo_config.broker_info
def setup_module():
"""
setUp
"""
global client
client = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
password=config.fe_password, http_port=config.fe_http_port)
timeout = 30
i = 0
variable_value = ""
# 设置default_rowset_type=beta
# 检查10次variable,每次等待30s
while i < 10:
# 由于是global变量,必须每请求一次换一个client
client_tmp = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
password=config.fe_password, http_port=config.fe_http_port)
set_variable_sql = "set global default_rowset_type=beta"
r = client_tmp.execute(set_variable_sql)
if variable_value == "beta":
time.sleep(timeout)
break
time.sleep(timeout)
variable_value = client_tmp.show_variables("default_rowset_type")[0][1]
i = i + 1
assert variable_value == "beta"
print("alter global variable default_rowset_type from alpha to beta!")
def check_partition_list(table_name, partition_name_list):
"""
验证分区是否创建成功
"""
for partition_name in partition_name_list:
assert client.get_partition(table_name, partition_name)
def partition_check(table_name, column_name, partition_name_list, \
partition_value_list, distribution_type, bucket_num, storage_type, schema, \
keys_desc=None, bitmap_index_list=None):
"""
检查,验证
"""
partition_info = palo_client.PartitionInfo(column_name, \
partition_name_list, partition_value_list)
distribution_info = palo_client.DistributionInfo(distribution_type, bucket_num)
client.create_table(table_name, schema, \
partition_info, distribution_info, keys_desc=keys_desc, bitmap_index_list=bitmap_index_list)
assert client.show_tables(table_name)
check_partition_list(table_name, partition_name_list)
def check(table_name, schema, keys_desc=None, bitmap_index_list=None):
"""
分区,检查
"""
partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d']
partition_value_list = ['5', '30', '100', 'MAXVALUE']
partition_check(table_name, 'k1',
partition_name_list, partition_value_list,
'HASH(k1, k2)', random.randrange(1, 30), 'column', schema, keys_desc,\
bitmap_index_list)
def test_create_index_when_create_table():
"""
{
"title": "test_sys_bitmap_index.test_create_index_when_create_table",
"describe": "test_create_index_when_create_table, 测试建表同时创建索引",
"tag": "system,p1"
}
"""
"""
test_create_index_when_create_table
测试建表同时创建索引
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table_with_index_col, bitmap_index_list=DATA.schema_agg_table_with_index)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'k1_idx', 'k1', table_name)
client.clean(database_name)
def test_create_index_after_create_table():
"""
{
"title": "test_sys_bitmap_index.test_create_index_after_create_table",
"describe": "test_create_index_after_create_table, 测试建表之后创建索引,使用两种语法创建:CREATE INDEX index_name ON table ... ;ALTER TABLE table ADD INDEX index_name ... ;",
"tag": "system,p1"
}
"""
"""
test_create_index_after_create_table
测试建表之后创建索引,使用两种语法创建:
CREATE INDEX index_name ON table ...
ALTER TABLE table ADD INDEX index_name ...
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_nameint', 'int_key',
is_wait=True)
time.sleep(10)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_name_char', 'char_key',
create_format=2, is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_nameint', 'int_key', table_name)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_name_char', 'char_key', table_name)
client.clean(database_name)
def test_create_index_with_data():
"""
{
"title": "test_sys_bitmap_index.test_create_index_with_data",
"describe": "test_create_index_with_data, 测试表中有数据时创建索引",
"tag": "system,p1"
}
"""
"""
test_create_index_with_data
测试表中有数据时创建索引
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table_simple)
sql_insert = "insert into %s values(1, 1, 1, 1, 1),(2, 2, 2, 2, 2)" % table_name
r = client_exe.execute(sql_insert)
assert r == ()
sql_select = "select * from %s" % table_name
ret = client_exe.execute(sql_select)
assert len(ret) == 2
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_k1', 'k1',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_k1', 'k1', table_name)
client.clean(database_name)
def test_delete_data_after_create_index():
"""
{
"title": "test_sys_bitmap_index.test_delete_data_after_create_index",
"describe": "test_delete_data_after_create_index, 测试创建索引后删除数据",
"tag": "system,p1"
}
"""
"""
test_delete_data_after_create_index
测试创建索引后删除数据
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table_simple)
sql_insert = "insert into %s values(1, 1, 1, 1, 1),(2, 2, 2, 2, 2)" % table_name
r = client_exe.execute(sql_insert)
assert r == ()
sql_select = "select * from %s" % table_name
ret = client_exe.execute(sql_select)
assert len(ret) == 2
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_k1', 'k1',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_k1', 'k1', table_name)
sql_insert = "delete from %s partition partition_a where k1 > 0" % table_name
r = client_exe.execute(sql_insert)
assert r == ()
sql_select = "select * from %s" % table_name
ret = client_exe.execute(sql_select)
assert len(ret) == 0
client.clean(database_name)
def test_drop_column_after_create_index():
"""
{
"title": "test_sys_bitmap_index.test_drop_column_after_create_index",
"describe": "test_drop_column_after_create_index, 测试创建索引后删除列,预期索引一并删除",
"tag": "system,p1,fuzz"
}
"""
"""
test_drop_column_after_create_index
测试创建索引后删除列,预期索引一并删除
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table_simple)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_k3', 'k3',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_k3', 'k3', table_name)
time.sleep(10)
# 删除列
util.assert_return(True, '',
client.schema_change, table_name, drop_column_list = ["k3"],
is_wait=True)
util.assert_return_flag(False,
client.is_exists_index_in_table, 'index_k3', 'k3', table_name)
client.clean(database_name)
def test_create_index_with_alltype_data():
"""
{
"title": "test_sys_bitmap_index.test_create_index_with_alltype_data",
"describe": "test_create_index_with_alltype_data, 测试各类支持的数据类型创建索引,包括TINYINT,SMALLINT,INT,BIGINT,CHAR,VARCHAR,DATE,DATETIME,LARGEINT,DECIMAL",
"tag": "system,p1"
}
"""
"""
test_create_index_with_alltype_data
测试各类支持的数据类型创建索引,包括TINYINT,SMALLINT,INT,
BIGINT,CHAR,VARCHAR,DATE,DATETIME,LARGEINT,DECIMAL
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_tinyint', 'k1',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_tinyint', 'k1', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_smallint', 'k2',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_smallint', 'k2', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_int', 'int_key',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_int', 'int_key', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_bigint', 'bigint_key',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_bigint', 'bigint_key', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_char_50_key', 'char_50_key',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_char_50_key', 'char_50_key', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_character_key', 'character_key',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_character_key', 'character_key', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_date_key', 'date_key',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_date_key', 'date_key', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_datetime_key', 'datetime_key',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_datetime_key', 'datetime_key', table_name)
client.clean(database_name)
def test_create_index_on_value_column():
"""
{
"title": "test_sys_bitmap_index.test_create_index_on_value_column",
"describe": "test_create_index_on_value_column, 测试在agg表的非key列创建index",
"tag": "system,p1,fuzz"
}
"""
"""
test_create_index_on_value_column
测试在agg表的非key列创建index
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table_simple)
util.assert_return(False, '',
client.create_bitmap_index_table, table_name, 'index_v1', 'v1',
is_wait=True)
client.clean(database_name)
def test_create_index_on_many_column():
"""
{
"title": "test_sys_bitmap_index.test_create_index_on_many_column",
"describe": "test_create_index_on_many_column,测试在多列创建索引(联合索引,目前不支持)",
"tag": "system,p1,fuzz"
}
"""
"""
test_create_index_on_many_column
测试在多列创建索引(联合索引,目前不支持)
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table_simple)
util.assert_return(False, '',
client.create_bitmap_index_table, table_name, 'index_k1_k2', 'k1,k2',
is_wait=True)
client.clean(database_name)
def test_create_index_repete():
"""
{
"title": "test_sys_bitmap_index.test_create_index_repete",
"describe": "test_create_index_repete,测试重复创建同一索引",
"tag": "system,p1,fuzz"
}
"""
"""
test_create_index_repete
测试重复创建同一索引
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_tinyint', 'k1',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_tinyint', 'k1', table_name)
util.assert_return(False, '',
client.create_bitmap_index_table, table_name, 'index_tinyint_new', 'k1',
is_wait=True)
client.clean(database_name)
def test_drop_index():
"""
{
"title": "test_sys_bitmap_index.test_drop_index",
"describe": "test_drop_index,测试删除索引",
"tag": "system,p1,fuzz"
}
"""
"""
test_drop_index
测试删除索引
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_tinyint', 'k1',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_tinyint', 'k1', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.drop_bitmap_index_table, table_name, 'index_tinyint', is_wait=True)
util.assert_return_flag(False,
client.is_exists_index_in_table, 'index_tinyint', 'k1', table_name)
client.clean(database_name)
def test_drop_index_not_exist():
"""
{
"title": "test_sys_bitmap_index.test_drop_index_not_exist",
"describe": "test_drop_index_not_exist,测试删除不存在的索引",
"tag": "system,p1,fuzz"
}
"""
"""
test_drop_index_not_exist
测试删除不存在的索引
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_tinyint', 'k1',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_tinyint', 'k1', table_name)
time.sleep(10)
util.assert_return_flag(True,
client.drop_bitmap_index_table, table_name, 'index_tinyint', is_wait=True)
util.assert_return_flag(False,
client.is_exists_index_in_table, 'index_tinyint', 'k1', table_name)
time.sleep(10)
util.assert_return(False, '',
client.drop_bitmap_index_table, table_name, 'index_tinyint', is_wait=True)
client.clean(database_name)
def test_load_result_corr():
"""
{
"title": "test_sys_bitmap_index.test_load_result_corr",
"describe": "test_load_result_corr,测试导入查询结果正确性",
"tag": "system,p1"
}
"""
"""
test_load_result_corr
测试导入查询结果正确性
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_agg_table, bitmap_index_list=DATA.schema_agg_table_index)
# 导入各类型数据
data_desc_list = palo_client.LoadDataInfo(DATA.file_path, table_name)
assert client.batch_load(util.get_label(), data_desc_list, max_filter_ratio=0.05, is_wait=True,
broker=broker_info)
assert client.verify(DATA.expected_data_file_list, table_name)
client.clean(database_name)
def test_create_index_on_dup_value_column():
"""
{
"title": "test_sys_bitmap_index.test_create_index_on_dup_value_column",
"describe": "test_create_index_on_dup_value_column,测试在dup表的非key列创建index",
"tag": "system,p1"
}
"""
"""
test_create_index_on_dup_value_column
测试在dup表的非key列创建index
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_dup, keys_desc=DATA.key_1_dup)
util.assert_return_flag(True,
client.create_bitmap_index_table, table_name, 'index_v1', 'v1',
is_wait=True)
util.assert_return_flag(True,
client.is_exists_index_in_table, 'index_v1', 'v1', table_name)
client.clean(database_name)
def test_create_index_on_dup_float_double_column():
"""
{
"title": "test_sys_bitmap_index.test_create_index_on_dup_float_double_column",
"describe": "test_create_index_on_dup_float_double_column, 测试在dup表的float,double列创建索引",
"tag": "system,p1,fuzz"
}
"""
"""
test_create_index_on_dup_float_double_column
测试在dup表的float,double列创建索引
"""
database_name, table_name, index_name = util.gen_num_format_name_list()
LOG.info(L('', database_name=database_name,
table_name=table_name, index_name=index_name))
client.clean(database_name)
client.create_database(database_name)
client_exe = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
database_name = database_name, password=config.fe_password, http_port=config.fe_http_port)
check(table_name, DATA.schema_dup, keys_desc=DATA.key_1_dup)
util.assert_return(False, '',
client.create_bitmap_index_table, table_name, 'index_v2', 'v2',
is_wait=True)
util.assert_return(False, '',
client.create_bitmap_index_table, table_name, 'index_v3', 'v3',
is_wait=True)
client.clean(database_name)
def teardown_module():
"""
tearDown
"""
client = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
password=config.fe_password, http_port=config.fe_http_port)
timeout = 30
i = 0
variable_value = ""
# 设置default_rowset_type=alpha
# 检查10次variable,每次等待30s
while i < 10:
# 由于是global变量,必须每请求一次换一个client
client_tmp = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user,
password=config.fe_password, http_port=config.fe_http_port)
set_variable_sql = "set global default_rowset_type=alpha"
r = client_tmp.execute(set_variable_sql)
if variable_value == "alpha":
time.sleep(timeout)
break
time.sleep(timeout)
variable_value = client_tmp.show_variables("default_rowset_type")[0][1]
i = i + 1
assert variable_value == "alpha"
print("alter global variable default_rowset_type from beta to alpha!")
print("End")
if __name__ == '__main__':
import pdb
pdb.set_trace()
setup_module()
test_create_index_when_create_table()
test_create_index_after_create_table()
test_create_index_with_data()
test_delete_data_after_create_index()
test_drop_column_after_create_index()
test_create_index_with_alltype_data()
test_create_index_on_value_column()
test_create_index_on_many_column()
test_create_index_repete()
test_drop_index()
test_drop_index_not_exist()
test_load_result_corr()
test_create_index_on_dup_value_column()
test_create_index_on_dup_float_double_column()
teardown_module()