pytest/sys/test_sys_hll_sc.py - doris - Git at Google

 #!/bin/env python
 # -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.

 ############################################################################
 #
 #   @file test_sys_load.py
 #   @date 2017-04-10 15:02:22
 #   @brief This file is a test file for palo small load in complex scenarios.
 #
 #############################################################################

 """
 测试hll data type
 """
 import sys
 import time
 import pytest

 sys.path.append("../")
 from lib import palo_config
 from lib import palo_client
 from lib import util
 from lib import palo_job
 import palo_logger
 import palo_exception


 client = None

 #日志 异常 对象
 LOG = palo_logger.Logger.getLogger()
 L = palo_logger.StructedLogMessage
 PaloClientException = palo_exception.PaloException

 config = palo_config.config
 compare = 'test_query_qa.test'


 def setup_module():
     """
     setUp
     """
     global client
     client = palo_client.PaloClient(config.fe_host, config.fe_query_port, user=config.fe_user, \
             password=config.fe_password)
     client.init()


 def wait_end(database_name):
     """
     wait to finished
     """
     ret = True
     print('waitint for load...')
     state = None
     while ret:
         job_list = client.get_load_job_list(database_name=database_name)
         state = palo_job.LoadJob(job_list[-1]).get_state()
         if state == "FINISHED" or state == "CANCELLED":
             print(state)
             ret = False
         time.sleep(1)
     assert state == "FINISHED"


 def execute(line):
     """execte sql"""
     print(line)
     palo_result = client.execute(line)
     print(palo_result)
     return palo_result


 def init(db_name, table_name):
     """
     create db, table, bulk load, batch load
     Args:
         db_name:
         table_name:
         create_sql:
         key_column:

     Returns:
     """
     line = 'DROP DATABASE IF EXISTS %s' % db_name
     client.execute(line)
     client.create_database(db_name)
     client.use(db_name)
     # client.execute('drop table if exists %s' % table_name)
     line = 'CREATE TABLE %s (\
     `id`  int COMMENT "", \
     `id1` tinyint COMMENT "", \
     `c_float` float SUM COMMENT "", \
     `hll_set` hll hll_union COMMENT "" \
     ) ENGINE=OLAP \
      DISTRIBUTED BY HASH(`id`, `id1`) BUCKETS 5 \
      PROPERTIES ( \
     "storage_type" = "COLUMN" \
     );' % table_name
     execute(line)
     line = 'insert into %s select k4, k1, k9, hll_hash(k9) from %s' % (table_name, compare)
     execute(line)
     wait_end(db_name)


 @pytest.mark.skip()
 def test_sc_add_hll_column():
     """
     {
     "title": "test_sys_hll_sc.test_sc_add_hll_column",
     "describe": "schema change, add column and drop",
     "tag": "autotest"
     }
     """
     """schema change, add column and drop"""
     db_name, table_name, invalied_name_1 = util.gen_name_list()
     init(db_name, table_name)
     # add column
     column = ['hll_add hll hll_union', 'hll_add_1 hll hll_union']
     ret = client.schema_change(table_name, add_column_list=column, is_wait=True)
     assert ret, 'add hll column failed'
     line = 'select hll_union_agg(hll_set), hll_union_agg(hll_add), hll_union_agg(hll_add_1) from %s' \
             % table_name
     ret1 = client.execute(line)
     assert ret1 == ((58797, 0, 0),)
     # drop column
     column = ['hll_add_1']
     ret = client.schema_change(table_name, drop_column_list=column, is_wait=True)
     assert ret, 'drop hll column failed'
     line = 'select hll_union_agg(hll_add), hll_union_agg(hll_add) from %s' % table_name
     ret2 = client.execute(line)
     assert ret1[0][2] == ret2[0][1]
     assert ret1[0][1] == ret2[0][0]
     client.clean(db_name)


 def test_sc_drop_hll_column():
     """
     {
     "title": "test_sys_hll_sc.test_sc_drop_hll_column",
     "describe": "schema change, drop column",
     "tag": "function,p1,fuzz"
     }
     """
     """schema change, drop column"""
     db_name, table_name, invalied_name_1 = util.gen_name_list()
     init(db_name, table_name)
     column = ['hll_set']
     ret = client.schema_change(table_name, drop_column_list=column, is_wait=True)
     assert ret, 'drop hll column failed'
     line = 'select hll_union_agg(hll_set) from %s' % table_name
     flag = 0
     try:
         print(line)
         ret = client.execute(line)
         flag = 1
     except Exception as e:
         print(str(e))
     assert flag == 0, 'expect select execute ok'
     line = 'insert into %s select k4, k1, k9 from %s' % (table_name, compare)
     execute(line)
     wait_end(db_name)
     line = 'select * from %s order by 1, 2, 3 limit 100' % table_name
     print(line)
     try:
         ret = client.execute(line)
     except Exception as e:
         print(str(e))
         assert 0 == 1
     line = 'select count(*) from %s ' % table_name
     ret = client.execute(line)
     assert int(ret[0][0]) > 0, 'select count result error'
     client.clean(db_name)


 def test_sc_modified_hll_column():
     """
     {
     "title": "test_sys_hll_sc.test_sc_modified_hll_column",
     "describe": "schema change, 将hll列变为其他类型/key",
     "tag": "function,p1,fuzz"
     }
     """
     """schema change, 将hll列变为其他类型/key"""
     db_name, table_name, invalied_name_1 = util.gen_name_list()
     init(db_name, table_name)
     client.use(db_name)
     # modify hll column, expect false
     column_list = ['hll_set hll default "0"', 'hll_set int replace default "0" after id1',
                    'hll_set int replace default "0"', 'hll_set varchar(256) replace default ""',
                    'hll_set varchar(256)  default "" after id1',
                    'hll_set char(256) replace default ""',
                    'hll_set bigint replace default ""', 'hll_set double replace default "0.01"',
                    'hll_set datetime replace default ""']
     for column in column_list:
         modify_column = [column, ]
         flag = 1
         try:
             ret = client.schema_change(table_name, modify_column_list=modify_column, is_wait=True)
             flag = 0
         except Exception as e:
             print(str(e))
         assert flag == 1

     order_list = ['id1', 'id', 'c_float', 'hll_set']
     ret = client.schema_change(table_name, order_column_list=order_list, is_wait=True)
     assert ret
     # check order
     ret = client.desc_table(table_name)
     assert ret[3][0] == 'hll_set'
     client.clean(db_name)


 def test_sc_rollup_hll_column():
     """
     {
     "title": "test_sys_hll_sc.test_sc_rollup_hll_column",
     "describe": "schema change, create rollup and delete rollup",
     "tag": "function,p1"
     }
     """
     """schema change, create rollup and delete rollup"""
     db_name, table_name, index_name = util.gen_name_list()
     init(db_name, table_name)
     line1 = 'select id1, id, hll_union_agg(hll_set) from %s group by id1, id order by 1, 2' \
             % table_name
     ret1_1 = client.execute(line1)
     line2 = 'select id1, hll_union_agg(hll_set) from %s group by id, id1 order by 1, 2' % table_name
     ret2_1 = client.execute(line2)
     index_name_derive = index_name + "_1"
     rollup_column = ['id1', 'id', 'hll_set']
     r = client.create_rollup_table(table_name, index_name, rollup_column, is_wait=True)
     assert r
     ret1_2 = client.execute(line1)
     # ret2_2 = client.execute(line2)
     util.check(ret1_1, ret1_2)

     rollup_column_1 = ['id1', 'hll_set']
     r = client.create_rollup_table(table_name, index_name_derive, rollup_column_1,
                                    base_index_name=index_name, is_wait=True)
     assert r
     ret3_2 = client.execute(line2)
     util.check(ret2_1, ret3_2)
     try:
         client.drop_rollup_table(table_name, index_name)
     except Exception as e:
         print(str(e))
         assert 0 == 1
     client.clean(db_name)


 if __name__ == '__main__':
     setup_module()
     # test_sc_add_hll_column()
     # test_sc_drop_hll_column()
     # test_sc_modified_hll_column()
     test_sc_rollup_hll_column()
	#!/bin/env python
	# -- coding: utf-8 --
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.

	############################################################################
	#
	# @file test_sys_load.py
	# @date 2017-04-10 15:02:22
	# @brief This file is a test file for palo small load in complex scenarios.
	#
	#############################################################################

	"""
	测试hll data type
	"""
	import sys
	import time
	import pytest

	sys.path.append("../")
	from lib import palo_config
	from lib import palo_client
	from lib import util
	from lib import palo_job
	import palo_logger
	import palo_exception


	client = None

	#日志异常对象
	LOG = palo_logger.Logger.getLogger()
	L = palo_logger.StructedLogMessage
	PaloClientException = palo_exception.PaloException

	config = palo_config.config
	compare = 'test_query_qa.test'


	def setup_module():
	"""
	setUp
	"""
	global client
	client = palo_client.PaloClient(config.fe_host, config.fe_query_port, user=config.fe_user, \
	password=config.fe_password)
	client.init()


	def wait_end(database_name):
	"""
	wait to finished
	"""
	ret = True
	print('waitint for load...')
	state = None
	while ret:
	job_list = client.get_load_job_list(database_name=database_name)
	state = palo_job.LoadJob(job_list[-1]).get_state()
	if state == "FINISHED" or state == "CANCELLED":
	print(state)
	ret = False
	time.sleep(1)
	assert state == "FINISHED"


	def execute(line):
	"""execte sql"""
	print(line)
	palo_result = client.execute(line)
	print(palo_result)
	return palo_result


	def init(db_name, table_name):
	"""
	create db, table, bulk load, batch load
	Args:
	db_name:
	table_name:
	create_sql:
	key_column:

	Returns:
	"""
	line = 'DROP DATABASE IF EXISTS %s' % db_name
	client.execute(line)
	client.create_database(db_name)
	client.use(db_name)
	# client.execute('drop table if exists %s' % table_name)
	line = 'CREATE TABLE %s (\
	`id` int COMMENT "", \
	`id1` tinyint COMMENT "", \
	`c_float` float SUM COMMENT "", \
	`hll_set` hll hll_union COMMENT "" \
	) ENGINE=OLAP \
	DISTRIBUTED BY HASH(`id`, `id1`) BUCKETS 5 \
	PROPERTIES ( \
	"storage_type" = "COLUMN" \
	);' % table_name
	execute(line)
	line = 'insert into %s select k4, k1, k9, hll_hash(k9) from %s' % (table_name, compare)
	execute(line)
	wait_end(db_name)


	@pytest.mark.skip()
	def test_sc_add_hll_column():
	"""
	{
	"title": "test_sys_hll_sc.test_sc_add_hll_column",
	"describe": "schema change, add column and drop",
	"tag": "autotest"
	}
	"""
	"""schema change, add column and drop"""
	db_name, table_name, invalied_name_1 = util.gen_name_list()
	init(db_name, table_name)
	# add column
	column = ['hll_add hll hll_union', 'hll_add_1 hll hll_union']
	ret = client.schema_change(table_name, add_column_list=column, is_wait=True)
	assert ret, 'add hll column failed'
	line = 'select hll_union_agg(hll_set), hll_union_agg(hll_add), hll_union_agg(hll_add_1) from %s' \
	% table_name
	ret1 = client.execute(line)
	assert ret1 == ((58797, 0, 0),)
	# drop column
	column = ['hll_add_1']
	ret = client.schema_change(table_name, drop_column_list=column, is_wait=True)
	assert ret, 'drop hll column failed'
	line = 'select hll_union_agg(hll_add), hll_union_agg(hll_add) from %s' % table_name
	ret2 = client.execute(line)
	assert ret1[0][2] == ret2[0][1]
	assert ret1[0][1] == ret2[0][0]
	client.clean(db_name)


	def test_sc_drop_hll_column():
	"""
	{
	"title": "test_sys_hll_sc.test_sc_drop_hll_column",
	"describe": "schema change, drop column",
	"tag": "function,p1,fuzz"
	}
	"""
	"""schema change, drop column"""
	db_name, table_name, invalied_name_1 = util.gen_name_list()
	init(db_name, table_name)
	column = ['hll_set']
	ret = client.schema_change(table_name, drop_column_list=column, is_wait=True)
	assert ret, 'drop hll column failed'
	line = 'select hll_union_agg(hll_set) from %s' % table_name
	flag = 0
	try:
	print(line)
	ret = client.execute(line)
	flag = 1
	except Exception as e:
	print(str(e))
	assert flag == 0, 'expect select execute ok'
	line = 'insert into %s select k4, k1, k9 from %s' % (table_name, compare)
	execute(line)
	wait_end(db_name)
	line = 'select * from %s order by 1, 2, 3 limit 100' % table_name
	print(line)
	try:
	ret = client.execute(line)
	except Exception as e:
	print(str(e))
	assert 0 == 1
	line = 'select count(*) from %s ' % table_name
	ret = client.execute(line)
	assert int(ret[0][0]) > 0, 'select count result error'
	client.clean(db_name)


	def test_sc_modified_hll_column():
	"""
	{
	"title": "test_sys_hll_sc.test_sc_modified_hll_column",
	"describe": "schema change, 将hll列变为其他类型/key",
	"tag": "function,p1,fuzz"
	}
	"""
	"""schema change, 将hll列变为其他类型/key"""
	db_name, table_name, invalied_name_1 = util.gen_name_list()
	init(db_name, table_name)
	client.use(db_name)
	# modify hll column, expect false
	column_list = ['hll_set hll default "0"', 'hll_set int replace default "0" after id1',
	'hll_set int replace default "0"', 'hll_set varchar(256) replace default ""',
	'hll_set varchar(256) default "" after id1',
	'hll_set char(256) replace default ""',
	'hll_set bigint replace default ""', 'hll_set double replace default "0.01"',
	'hll_set datetime replace default ""']
	for column in column_list:
	modify_column = [column, ]
	flag = 1
	try:
	ret = client.schema_change(table_name, modify_column_list=modify_column, is_wait=True)
	flag = 0
	except Exception as e:
	print(str(e))
	assert flag == 1

	order_list = ['id1', 'id', 'c_float', 'hll_set']
	ret = client.schema_change(table_name, order_column_list=order_list, is_wait=True)
	assert ret
	# check order
	ret = client.desc_table(table_name)
	assert ret[3][0] == 'hll_set'
	client.clean(db_name)


	def test_sc_rollup_hll_column():
	"""
	{
	"title": "test_sys_hll_sc.test_sc_rollup_hll_column",
	"describe": "schema change, create rollup and delete rollup",
	"tag": "function,p1"
	}
	"""
	"""schema change, create rollup and delete rollup"""
	db_name, table_name, index_name = util.gen_name_list()
	init(db_name, table_name)
	line1 = 'select id1, id, hll_union_agg(hll_set) from %s group by id1, id order by 1, 2' \
	% table_name
	ret1_1 = client.execute(line1)
	line2 = 'select id1, hll_union_agg(hll_set) from %s group by id, id1 order by 1, 2' % table_name
	ret2_1 = client.execute(line2)
	index_name_derive = index_name + "_1"
	rollup_column = ['id1', 'id', 'hll_set']
	r = client.create_rollup_table(table_name, index_name, rollup_column, is_wait=True)
	assert r
	ret1_2 = client.execute(line1)
	# ret2_2 = client.execute(line2)
	util.check(ret1_1, ret1_2)

	rollup_column_1 = ['id1', 'hll_set']
	r = client.create_rollup_table(table_name, index_name_derive, rollup_column_1,
	base_index_name=index_name, is_wait=True)
	assert r
	ret3_2 = client.execute(line2)
	util.check(ret2_1, ret3_2)
	try:
	client.drop_rollup_table(table_name, index_name)
	except Exception as e:
	print(str(e))
	assert 0 == 1
	client.clean(db_name)


	if __name__ == '__main__':
	setup_module()
	# test_sc_add_hll_column()
	# test_sc_drop_hll_column()
	# test_sc_modified_hll_column()
	test_sc_rollup_hll_column()