| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| import argparse |
| import concurrent.futures |
| from datafusion import SessionContext |
| import os |
| import pyarrow |
| import subprocess |
| import time |
| |
| table_names = [ |
| "call_center", |
| "catalog_page", |
| "catalog_sales", |
| "catalog_returns", |
| "customer", |
| "customer_address", |
| "customer_demographics", |
| "date_dim", |
| "income_band", |
| "household_demographics", |
| "inventory", |
| "store", |
| "ship_mode", |
| "reason", |
| "promotion", |
| "item", |
| "store_sales", |
| "store_returns", |
| "web_page", |
| "warehouse", |
| "time_dim", |
| "web_site", |
| "web_sales", |
| "web_returns", |
| ] |
| |
| all_schemas = {} |
| |
| # TODO set PK/FK to not nullable |
| |
| all_schemas['customer_address'] = [ |
| pyarrow.field("ca_address_sk", pyarrow.int32()), |
| pyarrow.field("ca_address_id", pyarrow.string()), |
| pyarrow.field("ca_street_number", pyarrow.string()), |
| pyarrow.field("ca_street_name", pyarrow.string()), |
| pyarrow.field("ca_street_type", pyarrow.string()), |
| pyarrow.field("ca_suite_number", pyarrow.string()), |
| pyarrow.field("ca_city", pyarrow.string()), |
| pyarrow.field("ca_county", pyarrow.string()), |
| pyarrow.field("ca_state", pyarrow.string()), |
| pyarrow.field("ca_zip", pyarrow.string()), |
| pyarrow.field("ca_country", pyarrow.string()), |
| pyarrow.field("ca_gmt_offset", pyarrow.decimal128(5, 2)), |
| pyarrow.field("ca_location_type", pyarrow.string()) |
| ] |
| |
| all_schemas['customer_demographics'] = [ |
| pyarrow.field("cd_demo_sk", pyarrow.int32()), |
| pyarrow.field("cd_gender", pyarrow.string()), |
| pyarrow.field("cd_marital_status", pyarrow.string()), |
| pyarrow.field("cd_education_status", pyarrow.string()), |
| pyarrow.field("cd_purchase_estimate", pyarrow.int32()), |
| pyarrow.field("cd_credit_rating", pyarrow.string()), |
| pyarrow.field("cd_dep_count", pyarrow.int32()), |
| pyarrow.field("cd_dep_employed_count", pyarrow.int32()), |
| pyarrow.field("cd_dep_college_count", pyarrow.int32()) |
| ] |
| |
| all_schemas['date_dim'] = [ |
| pyarrow.field("d_date_sk", pyarrow.int32()), |
| pyarrow.field("d_date_id", pyarrow.string()), |
| pyarrow.field("d_date", pyarrow.date32()), |
| pyarrow.field("d_month_seq", pyarrow.int32()), |
| pyarrow.field("d_week_seq", pyarrow.int32()), |
| pyarrow.field("d_quarter_seq", pyarrow.int32()), |
| pyarrow.field("d_year", pyarrow.int32()), |
| pyarrow.field("d_dow", pyarrow.int32()), |
| pyarrow.field("d_moy", pyarrow.int32()), |
| pyarrow.field("d_dom", pyarrow.int32()), |
| pyarrow.field("d_qoy", pyarrow.int32()), |
| pyarrow.field("d_fy_year", pyarrow.int32()), |
| pyarrow.field("d_fy_quarter_seq", pyarrow.int32()), |
| pyarrow.field("d_fy_week_seq", pyarrow.int32()), |
| pyarrow.field("d_day_name", pyarrow.string()), |
| pyarrow.field("d_quarter_name", pyarrow.string()), |
| pyarrow.field("d_holiday", pyarrow.string()), |
| pyarrow.field("d_weekend", pyarrow.string()), |
| pyarrow.field("d_following_holiday", pyarrow.string()), |
| pyarrow.field("d_first_dom", pyarrow.int32()), |
| pyarrow.field("d_last_dom", pyarrow.int32()), |
| pyarrow.field("d_same_day_ly", pyarrow.int32()), |
| pyarrow.field("d_same_day_lq", pyarrow.int32()), |
| pyarrow.field("d_current_day", pyarrow.string()), |
| pyarrow.field("d_current_week", pyarrow.string()), |
| pyarrow.field("d_current_month", pyarrow.string()), |
| pyarrow.field("d_current_quarter", pyarrow.string()), |
| pyarrow.field("d_current_year", pyarrow.string()), |
| ] |
| |
| all_schemas["warehouse"] = [ |
| pyarrow.field("w_warehouse_sk", pyarrow.int32()), |
| pyarrow.field("w_warehouse_id", pyarrow.string()), |
| pyarrow.field("w_warehouse_name", pyarrow.string()), |
| pyarrow.field("w_warehouse_sq_ft", pyarrow.int32()), |
| pyarrow.field("w_street_number", pyarrow.string()), |
| pyarrow.field("w_street_name", pyarrow.string()), |
| pyarrow.field("w_street_type", pyarrow.string()), |
| pyarrow.field("w_suite_number", pyarrow.string()), |
| pyarrow.field("w_city", pyarrow.string()), |
| pyarrow.field("w_county", pyarrow.string()), |
| pyarrow.field("w_state", pyarrow.string()), |
| pyarrow.field("w_zip", pyarrow.string()), |
| pyarrow.field("w_country", pyarrow.string()), |
| pyarrow.field("w_gmt_offset", pyarrow.decimal128(5, 2)), |
| ] |
| |
| all_schemas["ship_mode"] = [ |
| pyarrow.field("sm_ship_mode_sk", pyarrow.int32()), |
| pyarrow.field("sm_ship_mode_id", pyarrow.string()), |
| pyarrow.field("sm_type", pyarrow.string()), |
| pyarrow.field("sm_code", pyarrow.string()), |
| pyarrow.field("sm_carrier", pyarrow.string()), |
| pyarrow.field("sm_contract", pyarrow.string()), |
| ] |
| |
| all_schemas["time_dim"] = [ |
| pyarrow.field("t_time_sk", pyarrow.int32()), |
| pyarrow.field("t_time_id", pyarrow.string()), |
| pyarrow.field("t_time", pyarrow.int32()), |
| pyarrow.field("t_hour", pyarrow.int32()), |
| pyarrow.field("t_minute", pyarrow.int32()), |
| pyarrow.field("t_second", pyarrow.int32()), |
| pyarrow.field("t_am_pm", pyarrow.string()), |
| pyarrow.field("t_shift", pyarrow.string()), |
| pyarrow.field("t_sub_shift", pyarrow.string()), |
| pyarrow.field("t_meal_time", pyarrow.string()), |
| ] |
| |
| all_schemas["reason"] = [ |
| pyarrow.field("r_reason_sk", pyarrow.int32()), |
| pyarrow.field("r_reason_id", pyarrow.string()), |
| pyarrow.field("r_reason_desc", pyarrow.string()), |
| ] |
| |
| all_schemas["income_band"] = [ |
| pyarrow.field("ib_income_band_sk", pyarrow.int32()), |
| pyarrow.field("ib_lower_bound", pyarrow.int32()), |
| pyarrow.field("ib_upper_bound", pyarrow.int32()), |
| ] |
| |
| |
| all_schemas["item"] = [ |
| pyarrow.field("i_item_sk", pyarrow.int32()), |
| pyarrow.field("i_item_id", pyarrow.string()), |
| pyarrow.field("i_rec_start_date", pyarrow.date32()), |
| pyarrow.field("i_rec_end_date", pyarrow.date32()), |
| pyarrow.field("i_item_desc", pyarrow.string()), |
| pyarrow.field("i_current_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("i_wholesale_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("i_brand_id", pyarrow.int32()), |
| pyarrow.field("i_brand", pyarrow.string()), |
| pyarrow.field("i_class_id", pyarrow.int32()), |
| pyarrow.field("i_class", pyarrow.string()), |
| pyarrow.field("i_category_id", pyarrow.int32()), |
| pyarrow.field("i_category", pyarrow.string()), |
| pyarrow.field("i_manufact_id", pyarrow.int32()), |
| pyarrow.field("i_manufact", pyarrow.string()), |
| pyarrow.field("i_size", pyarrow.string()), |
| pyarrow.field("i_formulation", pyarrow.string()), |
| pyarrow.field("i_color", pyarrow.string()), |
| pyarrow.field("i_units", pyarrow.string()), |
| pyarrow.field("i_container", pyarrow.string()), |
| pyarrow.field("i_manager_id", pyarrow.int32()), |
| pyarrow.field("i_product_name", pyarrow.string()), |
| ] |
| |
| all_schemas["store"] = [ |
| pyarrow.field("s_store_sk", pyarrow.int32()), |
| pyarrow.field("s_store_id", pyarrow.string()), |
| pyarrow.field("s_rec_start_date", pyarrow.date32()), |
| pyarrow.field("s_rec_end_date", pyarrow.date32()), |
| pyarrow.field("s_closed_date_sk", pyarrow.int32()), |
| pyarrow.field("s_store_name", pyarrow.string()), |
| pyarrow.field("s_number_employees", pyarrow.int32()), |
| pyarrow.field("s_floor_space", pyarrow.int32()), |
| pyarrow.field("s_hours", pyarrow.string()), |
| pyarrow.field("s_manager", pyarrow.string()), |
| pyarrow.field("s_market_id", pyarrow.int32()), |
| pyarrow.field("s_geography_class", pyarrow.string()), |
| pyarrow.field("s_market_desc", pyarrow.string()), |
| pyarrow.field("s_market_manager", pyarrow.string()), |
| pyarrow.field("s_division_id", pyarrow.int32()), |
| pyarrow.field("s_division_name", pyarrow.string()), |
| pyarrow.field("s_company_id", pyarrow.int32()), |
| pyarrow.field("s_company_name", pyarrow.string()), |
| pyarrow.field("s_street_number", pyarrow.string()), |
| pyarrow.field("s_street_name", pyarrow.string()), |
| pyarrow.field("s_street_type", pyarrow.string()), |
| pyarrow.field("s_suite_number", pyarrow.string()), |
| pyarrow.field("s_city", pyarrow.string()), |
| pyarrow.field("s_county", pyarrow.string()), |
| pyarrow.field("s_state", pyarrow.string()), |
| pyarrow.field("s_zip", pyarrow.string()), |
| pyarrow.field("s_country", pyarrow.string()), |
| pyarrow.field("s_gmt_offset", pyarrow.decimal128(5, 2)), |
| pyarrow.field("s_tax_precentage", pyarrow.decimal128(5, 2)), |
| ] |
| |
| all_schemas["call_center"] = [ |
| pyarrow.field("cc_call_center_sk", pyarrow.int32()), |
| pyarrow.field("cc_call_center_id", pyarrow.string()), |
| pyarrow.field("cc_rec_start_date", pyarrow.date32()), |
| pyarrow.field("cc_rec_end_date", pyarrow.date32()), |
| pyarrow.field("cc_closed_date_sk", pyarrow.int32()), |
| pyarrow.field("cc_open_date_sk", pyarrow.int32()), |
| pyarrow.field("cc_name", pyarrow.string()), |
| pyarrow.field("cc_class", pyarrow.string()), |
| pyarrow.field("cc_employees", pyarrow.int32()), |
| pyarrow.field("cc_sq_ft", pyarrow.int32()), |
| pyarrow.field("cc_hours", pyarrow.string()), |
| pyarrow.field("cc_manager", pyarrow.string()), |
| pyarrow.field("cc_mkt_id", pyarrow.int32()), |
| pyarrow.field("cc_mkt_class", pyarrow.string()), |
| pyarrow.field("cc_mkt_desc", pyarrow.string()), |
| pyarrow.field("cc_market_manager", pyarrow.string()), |
| pyarrow.field("cc_division", pyarrow.int32()), |
| pyarrow.field("cc_division_name", pyarrow.string()), |
| pyarrow.field("cc_company", pyarrow.int32()), |
| pyarrow.field("cc_company_name", pyarrow.string()), |
| pyarrow.field("cc_street_number", pyarrow.string()), |
| pyarrow.field("cc_street_name", pyarrow.string()), |
| pyarrow.field("cc_street_type", pyarrow.string()), |
| pyarrow.field("cc_suite_number", pyarrow.string()), |
| pyarrow.field("cc_city", pyarrow.string()), |
| pyarrow.field("cc_county", pyarrow.string()), |
| pyarrow.field("cc_state", pyarrow.string()), |
| pyarrow.field("cc_zip", pyarrow.string()), |
| pyarrow.field("cc_country", pyarrow.string()), |
| pyarrow.field("cc_gmt_offset", pyarrow.decimal128(5, 2)), |
| pyarrow.field("cc_tax_percentage", pyarrow.decimal128(5, 2)), |
| ] |
| |
| all_schemas["customer"] = [ |
| pyarrow.field("c_customer_sk", pyarrow.int32()), |
| pyarrow.field("c_customer_id", pyarrow.string()), |
| pyarrow.field("c_current_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("c_current_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("c_current_addr_sk", pyarrow.int32()), |
| pyarrow.field("c_first_shipto_date_sk", pyarrow.int32()), |
| pyarrow.field("c_first_sales_date_sk", pyarrow.int32()), |
| pyarrow.field("c_salutation", pyarrow.string()), |
| pyarrow.field("c_first_name", pyarrow.string()), |
| pyarrow.field("c_last_name", pyarrow.string()), |
| pyarrow.field("c_preferred_cust_flag", pyarrow.string()), |
| pyarrow.field("c_birth_day", pyarrow.int32()), |
| pyarrow.field("c_birth_month", pyarrow.int32()), |
| pyarrow.field("c_birth_year", pyarrow.int32()), |
| pyarrow.field("c_birth_country", pyarrow.string()), |
| pyarrow.field("c_login", pyarrow.string()), |
| pyarrow.field("c_email_address", pyarrow.string()), |
| pyarrow.field("c_last_review_date_sk", pyarrow.string()), |
| ] |
| |
| all_schemas["web_site"] = [ |
| pyarrow.field("web_site_sk", pyarrow.int32()), |
| pyarrow.field("web_site_id", pyarrow.string()), |
| pyarrow.field("web_rec_start_date", pyarrow.date32()), |
| pyarrow.field("web_rec_end_date", pyarrow.date32()), |
| pyarrow.field("web_name", pyarrow.string()), |
| pyarrow.field("web_open_date_sk", pyarrow.int32()), |
| pyarrow.field("web_close_date_sk", pyarrow.int32()), |
| pyarrow.field("web_class", pyarrow.string()), |
| pyarrow.field("web_manager", pyarrow.string()), |
| pyarrow.field("web_mkt_id", pyarrow.int32()), |
| pyarrow.field("web_mkt_class", pyarrow.string()), |
| pyarrow.field("web_mkt_desc", pyarrow.string()), |
| pyarrow.field("web_market_manager", pyarrow.string()), |
| pyarrow.field("web_company_id", pyarrow.int32()), |
| pyarrow.field("web_company_name", pyarrow.string()), |
| pyarrow.field("web_street_number", pyarrow.string()), |
| pyarrow.field("web_street_name", pyarrow.string()), |
| pyarrow.field("web_street_type", pyarrow.string()), |
| pyarrow.field("web_suite_number", pyarrow.string()), |
| pyarrow.field("web_city", pyarrow.string()), |
| pyarrow.field("web_county", pyarrow.string()), |
| pyarrow.field("web_state", pyarrow.string()), |
| pyarrow.field("web_zip", pyarrow.string()), |
| pyarrow.field("web_country", pyarrow.string()), |
| pyarrow.field("web_gmt_offset", pyarrow.decimal128(5, 2)), |
| pyarrow.field("web_tax_percentage", pyarrow.decimal128(5, 2)), |
| ] |
| |
| all_schemas["store_returns"] = [ |
| pyarrow.field("sr_returned_date_sk", pyarrow.int32()), |
| pyarrow.field("sr_return_time_sk", pyarrow.int32()), |
| pyarrow.field("sr_item_sk", pyarrow.int32()), |
| pyarrow.field("sr_customer_sk", pyarrow.int32()), |
| pyarrow.field("sr_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("sr_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("sr_addr_sk", pyarrow.int32()), |
| pyarrow.field("sr_store_sk", pyarrow.int32()), |
| pyarrow.field("sr_reason_sk", pyarrow.int32()), |
| pyarrow.field("sr_ticket_number", pyarrow.int32()), |
| pyarrow.field("sr_return_quantity", pyarrow.int32()), |
| pyarrow.field("sr_return_amt", pyarrow.decimal128(7, 2)), |
| pyarrow.field("sr_return_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("sr_return_amt_inc_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("sr_fee", pyarrow.decimal128(7, 2)), |
| pyarrow.field("sr_return_ship_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("sr_refunded_cash", pyarrow.decimal128(7, 2)), |
| pyarrow.field("sr_reversed_charge", pyarrow.decimal128(7, 2)), |
| pyarrow.field("sr_store_credit", pyarrow.decimal128(7, 2)), |
| pyarrow.field("sr_net_loss", pyarrow.decimal128(7, 2)), |
| ] |
| |
| all_schemas["household_demographics"] = [ |
| pyarrow.field("hd_demo_sk", pyarrow.int32()), |
| pyarrow.field("hd_income_band_sk", pyarrow.int32()), |
| pyarrow.field("hd_buy_potential", pyarrow.string()), |
| pyarrow.field("hd_dep_count", pyarrow.int32()), |
| pyarrow.field("hd_vehicle_count", pyarrow.int32()), |
| ] |
| |
| all_schemas["web_page"] = [ |
| pyarrow.field("wp_web_page_sk", pyarrow.int32()), |
| pyarrow.field("wp_web_page_id", pyarrow.string()), |
| pyarrow.field("wp_rec_start_date", pyarrow.date32()), |
| pyarrow.field("wp_rec_end_date", pyarrow.date32()), |
| pyarrow.field("wp_creation_date_sk", pyarrow.int32()), |
| pyarrow.field("wp_access_date_sk", pyarrow.int32()), |
| pyarrow.field("wp_autogen_flag", pyarrow.string()), |
| pyarrow.field("wp_customer_sk", pyarrow.int32()), |
| pyarrow.field("wp_url", pyarrow.string()), |
| pyarrow.field("wp_type", pyarrow.string()), |
| pyarrow.field("wp_char_count", pyarrow.int32()), |
| pyarrow.field("wp_link_count", pyarrow.int32()), |
| pyarrow.field("wp_image_count", pyarrow.int32()), |
| pyarrow.field("wp_max_ad_count", pyarrow.int32()), |
| ] |
| |
| all_schemas["promotion"] = [ |
| pyarrow.field("p_promo_sk", pyarrow.int32()), |
| pyarrow.field("p_promo_id", pyarrow.string()), |
| pyarrow.field("p_start_date_sk", pyarrow.int32()), |
| pyarrow.field("p_end_date_sk", pyarrow.int32()), |
| pyarrow.field("p_item_sk", pyarrow.int32()), |
| pyarrow.field("p_cost", pyarrow.decimal128(15, 2)), |
| pyarrow.field("p_response_target", pyarrow.int32()), |
| pyarrow.field("p_promo_name", pyarrow.string()), |
| pyarrow.field("p_channel_dmail", pyarrow.string()), |
| pyarrow.field("p_channel_email", pyarrow.string()), |
| pyarrow.field("p_channel_catalog", pyarrow.string()), |
| pyarrow.field("p_channel_tv", pyarrow.string()), |
| pyarrow.field("p_channel_radio", pyarrow.string()), |
| pyarrow.field("p_channel_press", pyarrow.string()), |
| pyarrow.field("p_channel_event", pyarrow.string()), |
| pyarrow.field("p_channel_demo", pyarrow.string()), |
| pyarrow.field("p_channel_details", pyarrow.string()), |
| pyarrow.field("p_purpose", pyarrow.string()), |
| pyarrow.field("p_discount_active", pyarrow.string()), |
| ] |
| |
| all_schemas["catalog_page"] = [ |
| pyarrow.field("cp_catalog_page_sk", pyarrow.int32()), |
| pyarrow.field("cp_catalog_page_id", pyarrow.string()), |
| pyarrow.field("cp_start_date_sk", pyarrow.int32()), |
| pyarrow.field("cp_end_date_sk", pyarrow.int32()), |
| pyarrow.field("cp_department", pyarrow.string()), |
| pyarrow.field("cp_catalog_number", pyarrow.int32()), |
| pyarrow.field("cp_catalog_page_number", pyarrow.int32()), |
| pyarrow.field("cp_description", pyarrow.string()), |
| pyarrow.field("cp_type", pyarrow.string()), |
| ] |
| |
| all_schemas["inventory"] = [ |
| pyarrow.field("inv_date_sk", pyarrow.int32()), |
| pyarrow.field("inv_item_sk", pyarrow.int32()), |
| pyarrow.field("inv_warehouse_sk", pyarrow.int32()), |
| pyarrow.field("inv_quantity_on_hand", pyarrow.int32()), |
| ] |
| |
| all_schemas["catalog_returns"] = [ |
| pyarrow.field("cr_returned_date_sk", pyarrow.int32()), |
| pyarrow.field("cr_returned_time_sk", pyarrow.int32()), |
| pyarrow.field("cr_item_sk", pyarrow.int32()), |
| pyarrow.field("cr_refunded_customer_sk", pyarrow.int32()), |
| pyarrow.field("cr_refunded_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("cr_refunded_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("cr_refunded_addr_sk", pyarrow.int32()), |
| pyarrow.field("cr_returning_customer_sk", pyarrow.int32()), |
| pyarrow.field("cr_returning_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("cr_returning_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("cr_returning_addr_sk", pyarrow.int32()), |
| pyarrow.field("cr_call_center_sk", pyarrow.int32()), |
| pyarrow.field("cr_catalog_page_sk", pyarrow.int32()), |
| pyarrow.field("cr_ship_mode_sk", pyarrow.int32()), |
| pyarrow.field("cr_warehouse_sk", pyarrow.int32()), |
| pyarrow.field("cr_reason_sk", pyarrow.int32()), |
| pyarrow.field("cr_order_number", pyarrow.int32()), |
| pyarrow.field("cr_return_quantity", pyarrow.int32()), |
| pyarrow.field("cr_return_amount", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cr_return_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cr_return_amt_inc_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cr_fee", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cr_return_ship_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cr_refunded_cash", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cr_reversed_charge", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cr_store_credit", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cr_net_loss", pyarrow.decimal128(7, 2)), |
| ] |
| |
| all_schemas["web_returns"] = [ |
| pyarrow.field("wr_returned_date_sk", pyarrow.int32()), |
| pyarrow.field("wr_returned_time_sk", pyarrow.int32()), |
| pyarrow.field("wr_item_sk", pyarrow.int32()), |
| pyarrow.field("wr_refunded_customer_sk", pyarrow.int32()), |
| pyarrow.field("wr_refunded_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("wr_refunded_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("wr_refunded_addr_sk", pyarrow.int32()), |
| pyarrow.field("wr_returning_customer_sk", pyarrow.int32()), |
| pyarrow.field("wr_returning_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("wr_returning_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("wr_returning_addr_sk", pyarrow.int32()), |
| pyarrow.field("wr_web_page_sk", pyarrow.int32()), |
| pyarrow.field("wr_reason_sk", pyarrow.int32()), |
| pyarrow.field("wr_order_number", pyarrow.int32()), |
| pyarrow.field("wr_return_quantity", pyarrow.int32()), |
| pyarrow.field("wr_return_amt", pyarrow.decimal128(7, 2)), |
| pyarrow.field("wr_return_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("wr_return_amt_inc_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("wr_fee", pyarrow.decimal128(7, 2)), |
| pyarrow.field("wr_return_ship_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("wr_refunded_cash", pyarrow.decimal128(7, 2)), |
| pyarrow.field("wr_reversed_charge", pyarrow.decimal128(7, 2)), |
| pyarrow.field("wr_account_credit", pyarrow.decimal128(7, 2)), |
| pyarrow.field("wr_net_loss", pyarrow.decimal128(7, 2)), |
| ] |
| |
| all_schemas["web_sales"] = [ |
| pyarrow.field("ws_sold_date_sk", pyarrow.int32()), |
| pyarrow.field("ws_sold_time_sk", pyarrow.int32()), |
| pyarrow.field("ws_ship_date_sk", pyarrow.int32()), |
| pyarrow.field("ws_item_sk", pyarrow.int32()), |
| pyarrow.field("ws_bill_customer_sk", pyarrow.int32()), |
| pyarrow.field("ws_bill_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("ws_bill_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("ws_bill_addr_sk", pyarrow.int32()), |
| pyarrow.field("ws_ship_customer_sk", pyarrow.int32()), |
| pyarrow.field("ws_ship_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("ws_ship_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("ws_ship_addr_sk", pyarrow.int32()), |
| pyarrow.field("ws_web_page_sk", pyarrow.int32()), |
| pyarrow.field("ws_web_site_sk", pyarrow.int32()), |
| pyarrow.field("ws_ship_mode_sk", pyarrow.int32()), |
| pyarrow.field("ws_warehouse_sk", pyarrow.int32()), |
| pyarrow.field("ws_promo_sk", pyarrow.int32()), |
| pyarrow.field("ws_order_number", pyarrow.int32()), |
| pyarrow.field("ws_quantity", pyarrow.int32()), |
| pyarrow.field("ws_wholesale_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_list_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_sales_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_ext_discount_amt", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_ext_sales_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_ext_wholesale_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_ext_list_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_ext_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_coupon_amt", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_ext_ship_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_net_paid", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_net_paid_inc_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_net_paid_inc_ship", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_net_paid_inc_ship_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ws_net_profit", pyarrow.decimal128(7, 2)), |
| ] |
| |
| all_schemas["catalog_sales"] = [ |
| pyarrow.field("cs_sold_date_sk", pyarrow.int32()), |
| pyarrow.field("cs_sold_time_sk", pyarrow.int32()), |
| pyarrow.field("cs_ship_date_sk", pyarrow.int32()), |
| pyarrow.field("cs_bill_customer_sk", pyarrow.int32()), |
| pyarrow.field("cs_bill_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("cs_bill_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("cs_bill_addr_sk", pyarrow.int32()), |
| pyarrow.field("cs_ship_customer_sk", pyarrow.int32()), |
| pyarrow.field("cs_ship_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("cs_ship_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("cs_ship_addr_sk", pyarrow.int32()), |
| pyarrow.field("cs_call_center_sk", pyarrow.int32()), |
| pyarrow.field("cs_catalog_page_sk", pyarrow.int32()), |
| pyarrow.field("cs_ship_mode_sk", pyarrow.int32()), |
| pyarrow.field("cs_warehouse_sk", pyarrow.int32()), |
| pyarrow.field("cs_item_sk", pyarrow.int32()), |
| pyarrow.field("cs_promo_sk", pyarrow.int32()), |
| pyarrow.field("cs_order_number", pyarrow.int32()), |
| pyarrow.field("cs_quantity", pyarrow.int32()), |
| pyarrow.field("cs_wholesale_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_list_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_sales_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_ext_discount_amt", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_ext_sales_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_ext_wholesale_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_ext_list_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_ext_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_coupon_amt", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_ext_ship_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_net_paid", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_net_paid_inc_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_net_paid_inc_ship", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_net_paid_inc_ship_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("cs_net_profit", pyarrow.decimal128(7, 2)), |
| ] |
| |
| all_schemas["store_sales"] = [ |
| pyarrow.field("ss_sold_date_sk", pyarrow.int32()), |
| pyarrow.field("ss_sold_time_sk", pyarrow.int32()), |
| pyarrow.field("ss_item_sk", pyarrow.int32()), |
| pyarrow.field("ss_customer_sk", pyarrow.int32()), |
| pyarrow.field("ss_cdemo_sk", pyarrow.int32()), |
| pyarrow.field("ss_hdemo_sk", pyarrow.int32()), |
| pyarrow.field("ss_addr_sk", pyarrow.int32()), |
| pyarrow.field("ss_store_sk", pyarrow.int32()), |
| pyarrow.field("ss_promo_sk", pyarrow.int32()), |
| pyarrow.field("ss_ticket_number", pyarrow.int32()), |
| pyarrow.field("ss_quantity", pyarrow.int32()), |
| pyarrow.field("ss_wholesale_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_list_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_sales_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_ext_discount_amt", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_ext_sales_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_ext_wholesale_cost", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_ext_list_price", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_ext_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_coupon_amt", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_net_paid", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_net_paid_inc_tax", pyarrow.decimal128(7, 2)), |
| pyarrow.field("ss_net_profit", pyarrow.decimal128(7, 2)), |
| ] |
| |
| def run(cmd: str): |
| print(f"Executing: {cmd}") |
| subprocess.run(cmd, shell=True, check=True) |
| |
| def run_and_log_output(cmd: str, log_file: str): |
| print(f"Executing: {cmd}; writing output to {log_file}") |
| with open(log_file, "w") as file: |
| subprocess.run(cmd, shell=True, check=True, stdout=file, stderr=subprocess.STDOUT) |
| |
| def convert_dat_to_parquet(ctx: SessionContext, table: str, dat_filename: str, file_extension: str, parquet_filename: str): |
| print(f"Converting {dat_filename} to {parquet_filename} ...") |
| |
| table_schema = all_schemas[table].copy() |
| |
| # Pre-collect the output columns so we can ignore the null field we add |
| # in to handle the trailing | in the file |
| output_cols = [r.name for r in table_schema] |
| |
| # Trailing | requires extra field for in processing |
| table_schema.append(pyarrow.field("some_null", pyarrow.null(), nullable=True)) |
| |
| schema = pyarrow.schema(table_schema) |
| |
| df = ctx.read_csv(dat_filename, schema=schema, has_header=False, file_extension=file_extension, delimiter="|") |
| df = df.select_columns(*output_cols) |
| df.write_parquet(parquet_filename, compression="snappy") |
| |
| def generate_tpcds(scale_factor: int, partitions: int): |
| pass |
| |
| def convert_tpcds(scale_factor: int, partitions: int): |
| start_time = time.time() |
| ctx = SessionContext() |
| if partitions == 1: |
| # convert to parquet |
| for table in table_names: |
| convert_dat_to_parquet(ctx, table, f"data/{table}.dat", "dat", f"data/{table}.parquet") |
| else: |
| for table in table_names: |
| run(f"mkdir -p data/{table}.parquet") |
| for part in range(1, partitions + 1): |
| source_file = f"data/{table}.dat/part-{part}.dat" |
| if os.path.exists(source_file): |
| convert_dat_to_parquet(ctx, table, source_file, "dat", f"data/{table}.parquet/part{part}.parquet") |
| end_time = time.time() |
| print(f"Converted CSV to Parquet in {round(end_time - start_time, 2)} seconds") |
| |
| if __name__ == '__main__': |
| arg_parser = argparse.ArgumentParser() |
| subparsers = arg_parser.add_subparsers(dest='command', help='Available commands') |
| |
| parser_generate = subparsers.add_parser('generate', help='Generate TPC-DS CSV Data') |
| parser_generate.add_argument('--scale-factor', type=int, help='The scale factor') |
| parser_generate.add_argument('--partitions', type=int, help='The number of partitions') |
| |
| parser_convert = subparsers.add_parser('convert', help='Convert TPC-DS CSV Data to Parquet') |
| parser_convert.add_argument('--scale-factor', type=int, help='The scale factor') |
| parser_convert.add_argument('--partitions', type=int, help='The number of partitions') |
| |
| args = arg_parser.parse_args() |
| if args.command == 'generate': |
| generate_tpcds(args.scale_factor, args.partitions) |
| elif args.command == 'convert': |
| convert_tpcds(args.scale_factor, args.partitions) |
| else: |
| print("invalid subcommand") |
