| #!/bin/bash |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| set -euo pipefail |
| . $IMPALA_HOME/bin/report_build_error.sh |
| setup_report_build_error |
| |
| IMPALA_DATA=${IMPALA_HOME}/testdata/impala-data |
| TPC_DS_DATA=${IMPALA_DATA}/tpcds |
| |
| SCALE_FACTOR=1 |
| if [[ $# == 1 && $1 -gt 1 ]] |
| then |
| SCALE_FACTOR=$1 |
| TPC_DS_DATA=${TPC_DS_DATA}${SCALE_FACTOR} |
| fi |
| |
| TPC_DS_DIRNAME=tpc-ds-${IMPALA_TPC_DS_VERSION} |
| TPC_DS_HOME=${IMPALA_TOOLCHAIN}/${TPC_DS_DIRNAME} |
| |
| echo "Generating TPC-DS data into ${TPC_DS_DATA}" |
| # Delete any preexisting data or symlinks |
| rm -rf ${TPC_DS_DATA} |
| mkdir -p ${TPC_DS_DATA} |
| cd ${TPC_DS_DATA} |
| |
| # dsdgen uses fixed size buffers that cause bizarre issues if the path to the |
| # binary is too long. Workaround by symlinking. |
| rm -f ${TPC_DS_DIRNAME} |
| ln -s ${TPC_DS_HOME} ${TPC_DS_DIRNAME} |
| TPC_DS_DSDGEN=${TPC_DS_DIRNAME}/bin/dsdgen |
| |
| if [ ! -x ${TPC_DS_DSDGEN} ]; then |
| echo "Could not find TPC-DS data generator executable: ${TPC_DS_DSDGEN}" |
| exit 1 |
| fi |
| |
| if [ -t 1 ] |
| then |
| # Output is terminal, show progress verbosely |
| VERBOSITY='-verbose' |
| else |
| VERBOSITY='' |
| fi |
| |
| ${TPC_DS_DSDGEN} -force ${VERBOSITY} -scale ${SCALE_FACTOR} |
| # Impala expects each table to be in its own subdirectory. |
| for FILE in *.dat; do |
| FILE_DIR=${FILE%.dat} |
| mkdir -p ${FILE_DIR} |
| mv ${FILE} ${FILE_DIR} |
| done |