| #!/bin/env python |
| # -*- coding: utf-8 -*- |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| ############################################################################ |
| # |
| # @file test_sys_partition_load.py |
| # @date 2015/05/13 15:26:21 |
| # @brief This file is a test file for palo data loading and verifying. |
| # |
| ############################################################################# |
| |
| """ |
| 复合分区，导入操作测试 |
| """ |
| |
| import time |
| import threading |
| import pytest |
| from data import partition as DATA |
| from data import timestamp as DATA_TS |
| from lib import palo_config |
| from lib import palo_client |
| from lib import palo_task |
| from lib import util |
| |
| LOG = palo_client.LOG |
| L = palo_client.L |
| config = palo_config.config |
| broker_info = palo_config.broker_info |
| |
| |
| class LoadThread(threading.Thread): |
| """ |
| 导入线程 |
| """ |
| def __init__(self, database_name, data_desc_list, label, max_filter_ratio=None): |
| threading.Thread.__init__(self) |
| self.database_name = database_name |
| self.data_desc_list = data_desc_list |
| self.label = label |
| self.max_filter_ratio = max_filter_ratio |
| |
| def run(self): |
| """ |
| run |
| """ |
| thread_client = palo_client.get_client(config.fe_host, config.fe_query_port, self.database_name, |
| user=config.fe_user, password=config.fe_password, |
| http_port=config.fe_http_port) |
| thread_client.batch_load(self.label, self.data_desc_list, broker=broker_info, |
| max_filter_ratio=self.max_filter_ratio, is_wait=True) |
| |
| |
| def setup_module(): |
| """ |
| setUp |
| """ |
| global client |
| client = palo_client.get_client(config.fe_host, config.fe_query_port, user=config.fe_user, |
| password=config.fe_password, http_port=config.fe_http_port) |
| |
| |
| def check_partition_list(table_name, partition_name_list): |
| """ |
| 验证分区是否创建成功 |
| """ |
| for partition_name in partition_name_list: |
| assert client.get_partition(table_name, partition_name) |
| |
| |
| def check_load_and_verify(table_name, load_partition_list=None): |
| """ |
| 验证表是否创建成功，分区是否创建成功，导入数据，校验 |
| """ |
| data_desc_list = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, load_partition_list) |
| assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info) |
| assert client.verify(DATA.expected_data_file_list_1, table_name) |
| assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info) |
| assert client.verify(list(DATA.expected_data_file_list_1) * 2, table_name) |
| |
| |
| def partition_check(table_name, column_name, partition_name_list, \ |
| partition_value_list, distribution_type, bucket_num, storage_type): |
| """ |
| 检查，验证 |
| """ |
| partition_info = palo_client.PartitionInfo(column_name, \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo(distribution_type, bucket_num) |
| client.create_table(table_name, DATA.schema_1, partition_info, |
| distribution_info, keys_desc='AGGREGATE KEY (k1, k2, k3, k4, k5)') |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| |
| def test_load_without_partition(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_without_partition", |
| "describe": "导入，不指定分区, 带rollup表", |
| "tag": "system,p1" |
| } |
| """ |
| """ |
| 导入，不指定分区, 带rollup表 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '30', 'MAXVALUE'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| |
| column_name_list = ['k1', 'k5', 'v4', 'v5'] |
| assert client.create_rollup_table(table_name, \ |
| index_name, column_name_list, is_wait=True) |
| |
| check_load_and_verify(table_name) |
| client.clean(database_name) |
| |
| |
| def test_load_with_partition(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_with_partition", |
| "describe": "导入，指定分区", |
| "tag": "system,p1" |
| } |
| """ |
| """ |
| 导入，指定分区 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '31', 'MAXVALUE'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| check_load_and_verify(table_name, ['partition_a', 'partition_b', 'partition_c']) |
| client.clean(database_name) |
| |
| |
| def test_multi_loading(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_multi_loading", |
| "describe": "并行导入", |
| "tag": "system,p1" |
| } |
| """ |
| """ |
| 并行导入 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '31', 'MAXVALUE'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| |
| data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c']) |
| data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d']) |
| data_desc_list_3 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| data_desc_list_4 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1') |
| laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2') |
| laod_thread_3 = LoadThread(database_name, data_desc_list_3, 'label_3') |
| laod_thread_4 = LoadThread(database_name, data_desc_list_4, 'label_4') |
| laod_thread_1.start() |
| laod_thread_2.start() |
| laod_thread_3.start() |
| laod_thread_4.start() |
| laod_thread_1.join() |
| laod_thread_2.join() |
| laod_thread_3.join() |
| laod_thread_4.join() |
| |
| assert client.verify(list(DATA.expected_data_file_list_1) * 4, table_name) |
| client.clean(database_name) |
| |
| |
| def test_multi_loading_for_one_database(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_multi_loading_for_one_database", |
| "describe": "同一库中的单分区表和复合分区表同时进行导入, #相同label导两次，失败", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 1. 同一库中的单分区表和复合分区表同时进行导入 |
| 2. #相同label导两次，失败 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| single_table_name = 'table_1' |
| client.create_table(single_table_name, DATA.schema_1) |
| assert client.show_tables(single_table_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '31', 'MAXVALUE'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| |
| data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, single_table_name) |
| data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d']) |
| laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1') |
| laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2') |
| laod_thread_1.start() |
| laod_thread_2.start() |
| laod_thread_1.join() |
| laod_thread_2.join() |
| assert client.verify(list(DATA.expected_data_file_list_1), single_table_name) |
| assert client.verify(list(DATA.expected_data_file_list_1), table_name) |
| |
| #相同label导两次，失败 |
| ret = False |
| try: |
| ret = client.batch_load('label_2', data_desc_list_2, is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| client.clean(database_name) |
| |
| |
| def test_cancel_load(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_cancel_load", |
| "describe": "取消导入", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 取消导入 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '31', 'MAXVALUE'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| |
| data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c']) |
| data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d']) |
| data_desc_list_3 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| data_desc_list_4 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1') |
| laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2') |
| laod_thread_3 = LoadThread(database_name, data_desc_list_3, 'label_3') |
| laod_thread_4 = LoadThread(database_name, data_desc_list_4, 'label_4') |
| laod_thread_1.start() |
| laod_thread_2.start() |
| laod_thread_3.start() |
| laod_thread_4.start() |
| time.sleep(1) |
| try: |
| client.cancel_load('label_2') |
| client.cancel_load('label_3') |
| except Exception as e: |
| print(str(e)) |
| raise pytest.skip('can not cancel load') |
| finally: |
| laod_thread_1.join() |
| laod_thread_2.join() |
| laod_thread_3.join() |
| laod_thread_4.join() |
| |
| assert client.verify(list(DATA.expected_data_file_list_1) * 2, table_name) |
| client.clean(database_name) |
| |
| |
| def test_add_partition_while_loading(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_add_partition_while_loading", |
| "describe": "导入过程中，增加分区", |
| "tag": "system,p1" |
| } |
| """ |
| """ |
| 导入过程中，增加分区 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '31', '40'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| |
| data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c']) |
| data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d']) |
| data_desc_list_3 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| data_desc_list_4 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1') |
| laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2') |
| laod_thread_3 = LoadThread(database_name, data_desc_list_3, 'label_3') |
| laod_thread_4 = LoadThread(database_name, data_desc_list_4, 'label_4') |
| laod_thread_1.start() |
| client.add_partition(table_name, 'partition_e', '50') |
| laod_thread_2.start() |
| timeout = 30 |
| while not client.get_load_job_list(state="LOADING") and timeout > 0: |
| time.sleep(1) |
| timeout -= 1 |
| client.add_partition(table_name, 'partition_f', '60') |
| laod_thread_3.start() |
| client.add_partition(table_name, 'partition_g', '70') |
| laod_thread_4.start() |
| client.add_partition(table_name, 'partition_h', '80') |
| laod_thread_1.join() |
| laod_thread_2.join() |
| laod_thread_3.join() |
| laod_thread_4.join() |
| |
| assert client.verify(list(DATA.expected_data_file_list_1) * 4, table_name) |
| add_partition_list = ['partition_e', 'partition_f', 'partition_g', 'partition_h'] |
| #FIXME |
| #check_partition_list(table_name, partition_name_list.extend(add_partition_list)) |
| check_partition_list(table_name, partition_name_list + add_partition_list) |
| client.clean(database_name) |
| |
| |
| def test_load_while_selecting(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_while_selecting", |
| "describe": "查询的同时导入", |
| "tag": "system,p1" |
| } |
| """ |
| """ |
| 查询的同时导入 |
| """ |
| |
| class SelectTask(palo_task.PaloTask): |
| """ |
| 导入线程 |
| """ |
| def __init__(self, database_name, table_name): |
| self.database_name = database_name |
| self.table_name = table_name |
| self.thread_client = palo_client.PaloClient(palo_config.config.fe_host, \ |
| palo_config.config.fe_query_port, database_name) |
| self.thread_client.init() |
| |
| def do_task(self): |
| """ |
| run |
| """ |
| self.thread_client.select_all(self.table_name) |
| |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '31', 'MAXVALUE'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| |
| data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c']) |
| data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_a', 'partition_b', 'partition_c', 'partition_d']) |
| data_desc_list_3 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| data_desc_list_4 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| laod_thread_1 = LoadThread(database_name, data_desc_list_1, 'label_1') |
| laod_thread_2 = LoadThread(database_name, data_desc_list_2, 'label_2') |
| laod_thread_3 = LoadThread(database_name, data_desc_list_3, 'label_3') |
| laod_thread_4 = LoadThread(database_name, data_desc_list_4, 'label_4') |
| laod_thread_1.start() |
| laod_thread_2.start() |
| timeout = 30 |
| while not client.get_load_job_list(state="LOADING") and timeout > 0: |
| time.sleep(1) |
| timeout -= 1 |
| select_task = SelectTask(database_name, table_name) |
| select_thread = palo_task.TaskThread(select_task) |
| select_thread.start() |
| laod_thread_3.start() |
| laod_thread_4.start() |
| laod_thread_1.join() |
| laod_thread_2.join() |
| laod_thread_3.join() |
| laod_thread_4.join() |
| select_thread.stop() |
| |
| assert client.verify(list(DATA.expected_data_file_list_1) * 4, table_name) |
| client.clean(database_name) |
| |
| |
| def test_load_while_deleting(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_while_deleting", |
| "describe": "同时向不同的分区导入和删除", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 同时向不同的分区导入和删除 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '31', 'MAXVALUE'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| |
| data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| assert client.batch_load(util.get_label(), data_desc_list_1, is_wait=True, broker=broker_info) |
| assert client.verify(DATA.expected_data_file_list_1, table_name) |
| data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_b', 'partition_c']) |
| laod_thread_1 = LoadThread(database_name, data_desc_list_2, 'label_1', max_filter_ratio='0.5') |
| laod_thread_1.start() |
| timeout = 30 |
| while not client.get_load_job_list(state="LOADING") and timeout > 0: |
| time.sleep(1) |
| timeout -= 1 |
| client.delete(table_name, [('k1', '<', '5'),], 'partition_a') |
| laod_thread_1.join() |
| |
| assert client.verify([DATA.expected_file_10] * 2, table_name) |
| client.clean(database_name) |
| |
| |
| def test_load_while_deleting_to_same_partition(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_while_deleting_to_same_partition", |
| "describe": "同时向相同的分区导入和删除", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 同时向相同的分区导入和删除 |
| 流式导入一期：支持同时向相同的分区导入和删除 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['5', '20', '31', 'MAXVALUE'] |
| |
| partition_check(table_name, 'k1', \ |
| partition_name_list, partition_value_list, \ |
| 'RANDOM', 13, 'row') |
| |
| data_desc_list_1 = palo_client.LoadDataInfo(DATA.file_path_1, table_name) |
| assert client.batch_load(util.get_label(), data_desc_list_1, is_wait=True, broker=broker_info) |
| assert client.verify(DATA.expected_data_file_list_1, table_name) |
| data_desc_list_2 = palo_client.LoadDataInfo(DATA.file_path_1, \ |
| table_name, ['partition_b', 'partition_c']) |
| load_thread_1 = LoadThread(database_name, data_desc_list_2, 'label_1', max_filter_ratio='0.5') |
| load_thread_1.start() |
| timeout = 30 |
| while not client.get_load_job_list(state="LOADING") and timeout > 0: |
| time.sleep(1) |
| timeout -= 1 |
| ret = False |
| try: |
| ret = client.delete(table_name, [('k1', '<', '20'),], 'partition_b') |
| except: |
| pass |
| load_thread_1.join() |
| # assert not ret |
| assert ret |
| client.clean(database_name) |
| |
| |
| def test_load_set_with_partition(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_set_with_partition", |
| "describe": "SET, 导入，不指定分区", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| SET, 导入，不指定分区 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| client.create_table(table_name, DATA_TS.schema_1, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k3 = strftime("%Y-%m-%d %H:%M:%S", tmp_k3)', \ |
| 'k4 = strftime("%Y-%m-%d %H:%M:%S", tmp_k4)', \ |
| 'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \ |
| 'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \ |
| 'v6 = strftime("%Y-%m-%d %H:%M:%S", tmp_v6)', \ |
| 'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)'] |
| |
| column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', \ |
| 'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_1, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info, max_filter_ratio=0.05) |
| assert client.verify(DATA_TS.expected_data_file_list_1, table_name) |
| client.clean(database_name) |
| |
| |
| def test_load_set_with_null_data(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_set_with_null_data", |
| "describe": "SET, 导入，有空数据", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| SET, 导入，有空数据 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| client.create_table(table_name, DATA_TS.schema_1, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k3 = strftime("%Y-%m-%d %H:%M:%S", tmp_k3)', \ |
| 'k4 = strftime("%Y-%m-%d %H:%M:%S", tmp_k4)', \ |
| 'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \ |
| 'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \ |
| 'v6 = strftime("%Y-%m-%d %H:%M:%S", tmp_v6)', \ |
| 'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)'] |
| |
| column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', \ |
| 'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_2, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='0.05', is_wait=True, broker=broker_info) |
| assert client.verify(DATA_TS.expected_data_file_list_1, table_name) |
| client.clean(database_name) |
| |
| |
| def test_load_set_out_of_range(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_set_out_of_range", |
| "describe": "SET, 导入，有超过范围的数据", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| SET, 导入，有超过范围的数据 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| client.create_table(table_name, DATA_TS.schema_1, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k3 = strftime("%Y-%m-%d %H:%M:%S", tmp_k3)', \ |
| 'k4 = strftime("%Y-%m-%d %H:%M:%S", tmp_k4)', \ |
| 'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \ |
| 'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \ |
| 'v6 = strftime("%Y-%m-%d %H:%M:%S", tmp_v6)', \ |
| 'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)'] |
| |
| column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', \ |
| 'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_3, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='0.05', is_wait=True, broker=broker_info) |
| assert client.verify(DATA_TS.expected_data_file_list_1, table_name) |
| client.clean(database_name) |
| |
| |
| def test_load_set_align(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_load_set_align", |
| "describe": "SET, 导入，对齐", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| SET, 导入，对齐 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['-300000', '0', '300000', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_2, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = alignment_timestamp("hour", tmp_k1)', \ |
| 'k3 = alignment_timestamp("day", tmp_k3)', \ |
| 'k4 = alignment_timestamp("month", tmp_k4)', \ |
| 'k5 = alignment_timestamp("year", tmp_k5)', \ |
| 'v2 = alignment_timestamp("year", tmp_v2)', \ |
| 'v4 = alignment_timestamp("month", tmp_v4)', \ |
| 'v6 = alignment_timestamp("day", tmp_v6)', \ |
| 'v7 = alignment_timestamp("hour", tmp_v7)'] |
| |
| column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \ |
| 'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_4, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info, max_filter_ratio=0.05) |
| assert client.verify(DATA_TS.expected_data_file_list_2, table_name) |
| client.clean(database_name) |
| |
| |
| def load_set_align_out_range(): |
| """ |
| SET, 导入，对齐, 对于year month有超过范围数据 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['-300000', '0', '300000', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_2, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = alignment_timestamp("hour", tmp_k1)', \ |
| 'k3 = alignment_timestamp("day", tmp_k3)', \ |
| 'k4 = alignment_timestamp("month", tmp_k4)', \ |
| 'k5 = alignment_timestamp("year", tmp_k5)', \ |
| 'v2 = alignment_timestamp("year", tmp_v2)', \ |
| 'v4 = alignment_timestamp("month", tmp_v4)', \ |
| 'v6 = alignment_timestamp("day", tmp_v6)', \ |
| 'v7 = alignment_timestamp("hour", tmp_v7)'] |
| |
| column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \ |
| 'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_5, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, max_filter_ratio='0.5', is_wait=True, |
| broker=broker_info) |
| assert client.verify(DATA_TS.expected_data_file_list_2, table_name) |
| client.clean(database_name) |
| |
| |
| def test_datetime_timestamp(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_datetime_timestamp", |
| "describe": "datetime timestamp", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| datetime timestamp |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_3, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k3 = alignment_timestamp("hour", tmp_k3)', \ |
| 'k4 = alignment_timestamp("day", tmp_k4)', \ |
| 'k5 = strftime("%Y-%m-%d %H:%M:%S", tmp_k5)', \ |
| 'k6 = alignment_timestamp("month", tmp_k6)', \ |
| 'k7 = alignment_timestamp("year", tmp_k7)', \ |
| 'k8 = strftime("%Y-%m-%d %H:%M:%S", tmp_k8)', \ |
| 'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \ |
| 'v3 = alignment_timestamp("year", tmp_v3)', \ |
| 'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \ |
| 'v5 = alignment_timestamp("month", tmp_v5)', \ |
| 'v6 = alignment_timestamp("day", tmp_v6)', \ |
| 'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)', \ |
| 'v8 = alignment_timestamp("hour", tmp_v8)', \ |
| 'v9 = strftime("%Y-%m-%d %H:%M:%S", tmp_v9)'] |
| |
| column_name_list = [\ |
| 'tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \ |
| 'tmp_k6', 'tmp_k7', 'tmp_k8', \ |
| 'v1', 'tmp_v2', 'tmp_v3', 'tmp_v4', 'tmp_v5', \ |
| 'tmp_v6', 'tmp_v7', 'tmp_v8', 'tmp_v9'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_6, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| assert client.verify(DATA_TS.expected_data_file_list_3, table_name) |
| client.clean(database_name) |
| |
| |
| def test_to_default_value_column(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_to_default_value_column", |
| "describe": "函数的返回值列为schema中指定默认值的列", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 函数的返回值列为schema中指定默认值的列 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_4, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k3 = alignment_timestamp("hour", tmp_k3)', \ |
| 'k4 = alignment_timestamp("day", tmp_k4)', \ |
| 'k5 = strftime("%Y-%m-%d %H:%M:%S", tmp_k5)', \ |
| 'k6 = alignment_timestamp("month", tmp_k6)', \ |
| 'k7 = alignment_timestamp("year", tmp_k7)', \ |
| 'k8 = strftime("%Y-%m-%d %H:%M:%S", tmp_k8)', \ |
| 'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \ |
| 'v3 = alignment_timestamp("year", tmp_v3)', \ |
| 'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \ |
| 'v5 = alignment_timestamp("month", tmp_v5)', \ |
| 'v6 = alignment_timestamp("day", tmp_v6)', \ |
| 'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)', \ |
| 'v8 = alignment_timestamp("hour", tmp_v8)', \ |
| 'v9 = strftime("%Y-%m-%d %H:%M:%S", tmp_v9)'] |
| |
| column_name_list = [\ |
| 'tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \ |
| 'tmp_k6', 'tmp_k7', 'tmp_k8', \ |
| 'v1', 'tmp_v2', 'tmp_v3', 'tmp_v4', 'tmp_v5', \ |
| 'tmp_v6', 'tmp_v7', 'tmp_v8', 'tmp_v9'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_6, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info, |
| max_filter_ratio=0.1) |
| assert client.verify(DATA_TS.expected_data_file_list_3, table_name) |
| client.clean(database_name) |
| |
| |
| def test_datetime_timestamp_invalid(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_datetime_timestamp_invalid", |
| "describe": "数据文件中时间戳列含有无效字符串、八进制、十六进制字符串、算术表达式、小数", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 数据文件中时间戳列含有无效字符串、八进制、十六进制字符串、算术表达式、小数 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_3, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k3 = alignment_timestamp("hour", tmp_k3)', \ |
| 'k4 = alignment_timestamp("day", tmp_k4)', \ |
| 'k5 = strftime("%Y-%m-%d %H:%M:%S", tmp_k5)', \ |
| 'k6 = alignment_timestamp("month", tmp_k6)', \ |
| 'k7 = alignment_timestamp("year", tmp_k7)', \ |
| 'k8 = strftime("%Y-%m-%d %H:%M:%S", tmp_k8)', \ |
| 'v2 = strftime("%Y-%m-%d %H:%M:%S", tmp_v2)', \ |
| 'v3 = alignment_timestamp("year", tmp_v3)', \ |
| 'v4 = strftime("%Y-%m-%d %H:%M:%S", tmp_v4)', \ |
| 'v5 = alignment_timestamp("month", tmp_v5)', \ |
| 'v6 = alignment_timestamp("day", tmp_v6)', \ |
| 'v7 = strftime("%Y-%m-%d %H:%M:%S", tmp_v7)', \ |
| 'v8 = alignment_timestamp("hour", tmp_v8)', \ |
| 'v9 = strftime("%Y-%m-%d %H:%M:%S", tmp_v9)'] |
| |
| column_name_list = [\ |
| 'tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \ |
| 'tmp_k6', 'tmp_k7', 'tmp_k8', \ |
| 'v1', 'tmp_v2', 'tmp_v3', 'tmp_v4', 'tmp_v5', \ |
| 'tmp_v6', 'tmp_v7', 'tmp_v8', 'tmp_v9'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_7, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| assert client.verify(DATA_TS.expected_data_file_list_3, table_name) |
| client.clean(database_name) |
| |
| |
| def timestamp_special_data(): |
| """ |
| 特殊数据 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_5, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| assert client.verify(DATA_TS.expected_data_file_list_4, table_name) |
| client.clean(database_name) |
| |
| |
| def test_invalid_parameter_part_1(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_invalid_parameter_part_1", |
| "describe": "函数缺少第1或2个参数，或同时缺少两个参数", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 函数缺少第1或2个参数，或同时缺少两个参数 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_5, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list_1 = ['k1 = strftime("%Y-%m-%d %H:%M:%S")', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| set_list_2 = ['k1 = strftime(tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| set_list_3 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp(tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| ret = False |
| column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_1) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_2) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_3) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| client.clean(database_name) |
| |
| |
| def test_invalid_parameter_part_2(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_invalid_parameter_part_2", |
| "describe": "函数缺少第1或2个参数，或同时缺少两个参数", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 函数缺少第1或2个参数，或同时缺少两个参数 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_5, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list_4 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year")'] |
| |
| set_list_5 = ['k1 = strftime()', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| set_list_6 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp()'] |
| |
| ret = False |
| column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_4) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_5) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_6) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| client.clean(database_name) |
| |
| |
| def test_invalid_parameter_part_3(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_invalid_parameter_part_3", |
| "describe": "函数传递空串""或"NULL"", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 函数传递空串""或"NULL" |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_5, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list_1 = ['k1 = strftime("", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| set_list_2 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", "")', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| set_list_3 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("NULL", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| ret = False |
| column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_1) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_2) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_3) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| client.clean(database_name) |
| |
| |
| def test_invalid_parameter_part_4(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_invalid_parameter_part_4", |
| "describe": "函数传递空串""或"NULL", 或无效的时间格式字符串参数", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 函数传递空串""或"NULL", 或无效的时间格式字符串参数 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_5, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list_4 = ['k1 = strftime(None, tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| set_list_5 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp(None, tmp_k3)'] |
| set_list_6 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", None)'] |
| |
| set_list_7 = ['k1 = strftime("%k-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| ret = False |
| column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_4) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_5) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_6) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_7) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| client.clean(database_name) |
| |
| |
| def test_invalid_parameter_part_5(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_invalid_parameter_part_5", |
| "describe": "不指定返回值或返回值列不存在", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 不指定返回值或返回值列不存在 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_5, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list_1 = ['strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| set_list_2 = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k5 = alignment_timestamp("year", tmp_k3)'] |
| ret = False |
| column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_1) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list_2) |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| client.clean(database_name) |
| |
| |
| def test_timestamp_invalid_format(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_timestamp_invalid_format", |
| "describe": "函数返回值列为datetime类型，但是不是Palo支持的格式", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 函数返回值列为datetime类型，但是不是Palo支持的格式 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_5, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y:%m:%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("month", tmp_k2)', \ |
| 'k3 = alignment_timestamp("year", tmp_k3)'] |
| |
| column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert not client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='0', is_wait=True, broker=broker_info) |
| client.clean(database_name) |
| |
| |
| def test_timestamp_upper_lower(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_timestamp_upper_lower", |
| "describe": "year|month|day|hour大小写", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| year|month|day|hour大小写 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_5, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", tmp_k1)', \ |
| 'k2 = alignment_timestamp("MONTH", tmp_k2)', \ |
| 'k3 = alignment_timestamp("YEAR", tmp_k3)'] |
| |
| column_name_list = ['tmp_k1', 'tmp_k2', 'tmp_k3'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_8, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert not client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='0', is_wait=True, broker=broker_info) |
| client.clean(database_name) |
| |
| |
| def test_no_column_list(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_no_column_list", |
| "describe": "不指定column_list", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 不指定column_list |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['1900-01-01 23:59:59', '2000-01-01 00:00:00', \ |
| '2099-12-21 23:59:59', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_3, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", k1)', \ |
| 'k3 = alignment_timestamp("hour", k3)', \ |
| 'k4 = alignment_timestamp("day", k4)', \ |
| 'k5 = strftime("%Y-%m-%d %H:%M:%S", k5)', \ |
| 'k6 = alignment_timestamp("month", k6)', \ |
| 'k7 = alignment_timestamp("year", k7)', \ |
| 'k8 = strftime("%Y-%m-%d %H:%M:%S", k8)', \ |
| 'v2 = strftime("%Y-%m-%d %H:%M:%S", v2)', \ |
| 'v3 = alignment_timestamp("year", v3)', \ |
| 'v4 = strftime("%Y-%m-%d %H:%M:%S", v4)', \ |
| 'v5 = alignment_timestamp("month", v5)', \ |
| 'v6 = alignment_timestamp("day", v6)', \ |
| 'v7 = strftime("%Y-%m-%d %H:%M:%S", v7)', \ |
| 'v8 = alignment_timestamp("hour", v8)', \ |
| 'v9 = strftime("%Y-%m-%d %H:%M:%S", v9)'] |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_6, \ |
| table_name, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| assert client.verify(DATA_TS.expected_data_file_list_3, table_name) |
| client.clean(database_name) |
| |
| |
| def test_to_varchar(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_to_varchar", |
| "describe": "时间戳转换成字符串类型", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 时间戳转换成字符串类型 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| assert client.create_table(table_name, DATA_TS.schema_6) |
| assert client.show_tables(table_name) |
| |
| set_list = ['k1 = strftime("%Y-%m-%d %H:%M:%S", k1)', \ |
| 'k3 = alignment_timestamp("hour", k3)', \ |
| 'k4 = alignment_timestamp("day", k4)', \ |
| 'k5 = strftime("%Y-%m-%d %H:%M:%S", k5)', \ |
| 'k6 = alignment_timestamp("month", k6)', \ |
| 'k7 = alignment_timestamp("year", k7)', \ |
| 'k8 = strftime("%Y-%m-%d %H:%M:%S", k8)', \ |
| 'v2 = strftime("%Y-%m-%d %H:%M:%S", v2)', \ |
| 'v3 = alignment_timestamp("year", v3)', \ |
| 'v4 = strftime("%Y-%m-%d %H:%M:%S", v4)', \ |
| 'v5 = alignment_timestamp("month", v5)', \ |
| 'v6 = alignment_timestamp("day", v6)', \ |
| 'v7 = strftime("%Y-%m-%d %H:%M:%S", v7)', \ |
| 'v8 = alignment_timestamp("hour", v8)', \ |
| 'v9 = strftime("%Y-%m-%d %H:%M:%S", v9)'] |
| |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_6, \ |
| table_name, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, \ |
| max_filter_ratio='1', is_wait=True, broker=broker_info) |
| assert client.verify(DATA_TS.expected_data_file_list_3, table_name) |
| client.clean(database_name) |
| |
| |
| def test_one_to_one(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_one_to_one", |
| "describe": "同一列多次转换或对齐都对应到表中的同一列", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 同一列多次转换或对齐都对应到表中的同一列 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['-300000', '0', '300000', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_2, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = alignment_timestamp("hour", tmp_k1)', \ |
| 'k3 = alignment_timestamp("day", tmp_k3)', \ |
| 'k4 = alignment_timestamp("month", tmp_k4)', \ |
| 'k5 = alignment_timestamp("year", tmp_k5)', \ |
| 'k5 = alignment_timestamp("year", tmp_k5)', \ |
| 'v2 = alignment_timestamp("year", tmp_v2)', \ |
| 'v4 = alignment_timestamp("month", tmp_v4)', \ |
| 'v6 = alignment_timestamp("day", tmp_v6)', \ |
| 'v7 = alignment_timestamp("hour", tmp_v7)'] |
| |
| column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \ |
| 'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_4, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| ret = False |
| try: |
| ret = client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info) |
| except: |
| pass |
| assert not ret |
| client.clean(database_name) |
| |
| |
| def test_one_to_many(): |
| """ |
| { |
| "title": "test_sys_partition_load.test_one_to_many", |
| "describe": "同一列转换后对应到表中的多列，及多个函数具有相同的参数列，不同的返回值列", |
| "tag": "system,p1,fuzz" |
| } |
| """ |
| """ |
| 同一列转换后对应到表中的多列，及多个函数具有相同的参数列，不同的返回值列 |
| """ |
| database_name, table_name, index_name = util.gen_num_format_name_list() |
| LOG.info(L('', database_name=database_name, \ |
| table_name=table_name, index_name=index_name)) |
| client.clean(database_name) |
| client.create_database(database_name) |
| |
| partition_name_list = ['partition_a', 'partition_b', 'partition_c', 'partition_d'] |
| partition_value_list = ['-300000', '0', '300000', 'MAXVALUE'] |
| partition_info = palo_client.PartitionInfo('k1', \ |
| partition_name_list, partition_value_list) |
| distribution_info = palo_client.DistributionInfo('RANDOM', 13) |
| assert client.create_table(table_name, DATA_TS.schema_7, \ |
| partition_info, distribution_info) |
| assert client.show_tables(table_name) |
| check_partition_list(table_name, partition_name_list) |
| |
| set_list = ['k1 = alignment_timestamp("hour", tmp_k1)', \ |
| 'k3 = alignment_timestamp("day", tmp_k3)', \ |
| 'k4 = alignment_timestamp("month", tmp_k4)', \ |
| 'k5 = alignment_timestamp("year", tmp_k5)', \ |
| 'v2 = alignment_timestamp("year", tmp_v2)', \ |
| 'v4 = alignment_timestamp("month", tmp_v4)', \ |
| 'v6 = alignment_timestamp("day", tmp_v6)', \ |
| 'v7 = alignment_timestamp("day", tmp_v6)'] |
| |
| column_name_list = ['tmp_k1', 'k2', 'tmp_k3', 'tmp_k4', 'tmp_k5', \ |
| 'v1', 'tmp_v2', 'v3', 'tmp_v4', 'v5', 'tmp_v6', 'tmp_v7'] |
| data_desc_list = palo_client.LoadDataInfo(DATA_TS.file_path_4, \ |
| table_name, column_name_list=column_name_list, set_list=set_list) |
| assert client.batch_load(util.get_label(), data_desc_list, is_wait=True, broker=broker_info, max_filter_ratio=0.05) |
| assert client.verify(DATA_TS.expected_data_file_list_5, table_name) |
| client.clean(database_name) |
| |
| |
| def teardown_module(): |
| """ |
| tearDown |
| """ |
| pass |
| |
| |
| if __name__ == '__main__': |
| import pdb |
| pdb.set_trace() |
| setup_module() |
| |
