PARQUET-679: [C++] Resolve unit tests issues on Windows; Run unit tes…
…ts with Appveyor; Appveyor build jobs for Visual Studio cmake generator and Debug configuration.
Author: Max Risuhin <risuhin.max@gmail.com>
Closes #319 from MaxRis/PARQUET-679 and squashes the following commits:
6bf2cd7 [Max Risuhin] PARQUET-679: [C++] Resolve unit tests issues on Windows; Run unit tests with Appveyor; Appveyor build jobs for Visual Studio cmake generator and Debug configuration.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 46d4394..e18cf7b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -153,6 +153,17 @@
# where to put generated binaries
set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+if (MSVC)
+ # define output directories to overwrite default Visual Studio generator output directories
+ FOREACH(BUILD_CONFIG ${CMAKE_CONFIGURATION_TYPES})
+ string(TOUPPER ${BUILD_CONFIG} UPPERCASE_BUILD_CONFIG)
+ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
+ set(CMAKE_PDB_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
+ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
+ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_${UPPERCASE_BUILD_CONFIG} ${BUILD_OUTPUT_ROOT_DIRECTORY})
+ ENDFOREACH()
+endif()
+
############################################################
# Benchmarking
############################################################
@@ -353,6 +364,9 @@
# Debug symbols are stripped for reduced binary size. Add
# -DPARQUET_CXXFLAGS="-g" to include them
set(CXX_FLAGS_DEBUG "-ggdb -O0")
+if (MSVC)
+ set(CXX_FLAGS_DEBUG "${CXX_FLAGS_DEBUG} /bigobj")
+endif()
set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
set(CXX_FLAGS_RELEASE "-O3")
diff --git a/appveyor.yml b/appveyor.yml
index 437cd6d..f3fac11 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -23,14 +23,22 @@
- GENERATOR: NMake Makefiles
PYTHON: "3.5"
ARCH: "64"
+
+ - GENERATOR: Visual Studio 14 2015 Win64
+ PYTHON: "3.5"
+ ARCH: "64"
MSVC_DEFAULT_OPTIONS: ON
BOOST_ROOT: C:\Libraries\boost_1_63_0
BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
+configuration:
+ - Debug
+ - Release
+
init:
- set MINICONDA=C:\Miniconda35-x64
- set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
- if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64
build_script:
- - call ci\msvc-build.bat
\ No newline at end of file
+ - call ci\msvc-build.bat
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index 7e723fa..b1b54b4 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -20,8 +20,16 @@
mkdir build
cd build
+SET PARQUET_TEST_DATA=%APPVEYOR_BUILD_FOLDER%\data
+
cmake -G "%GENERATOR%" ^
- -DCMAKE_BUILD_TYPE=Release ^
+ -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+ -DPARQUET_BOOST_USE_SHARED=OFF ^
+ -DPARQUET_CXXFLAGS="/MP" ^
.. || exit /B
-nmake || exit /B
\ No newline at end of file
+cmake --build . --config %CONFIGURATION% || exit /B
+
+if "%CONFIGURATION%" == "Release" (
+ ctest -VV || exit /B
+)
\ No newline at end of file
diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake
index 44b2619..74b4632 100644
--- a/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cmake_modules/ThirdpartyToolchain.cmake
@@ -82,7 +82,7 @@
endif()
find_package(Boost COMPONENTS regex REQUIRED)
- if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+ if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
else()
set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
@@ -91,7 +91,7 @@
# Find static Boost libraries.
set(Boost_USE_STATIC_LIBS ON)
find_package(Boost COMPONENTS regex REQUIRED)
- if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+ if ("${UPPERCASE_BUILD_TYPE}" STREQUAL "DEBUG")
set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
else()
set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
@@ -129,7 +129,11 @@
set(ZLIB_HOME "${ZLIB_PREFIX}")
set(ZLIB_INCLUDE_DIRS "${ZLIB_PREFIX}/include")
if (MSVC)
- set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
+ if (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+ set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib)
+ else()
+ set(ZLIB_STATIC_LIB_NAME zlibstatic.lib)
+ endif()
else()
set(ZLIB_STATIC_LIB_NAME libz.a)
endif()
@@ -175,7 +179,11 @@
set(THRIFT_HOME "${THRIFT_PREFIX}")
set(THRIFT_INCLUDE_DIR "${THRIFT_PREFIX}/include")
IF (${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
- set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthriftd.a")
+ IF (MSVC)
+ set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/thriftmdd.lib")
+ ELSE()
+ set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/libthriftd.a")
+ ENDIF()
ELSE()
IF (MSVC)
set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/thriftmd.lib")
@@ -206,7 +214,9 @@
"-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIRS}"
"-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}"
"-DWITH_SHARED_LIB=OFF"
+ "-DWITH_PLUGIN=OFF"
${THRIFT_CMAKE_ARGS})
+ set (THRIFT_DEPENDENCIES zlib_ep)
endif()
if (CMAKE_VERSION VERSION_GREATER "3.2")
@@ -215,12 +225,14 @@
URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"
BUILD_BYPRODUCTS "${THRIFT_STATIC_LIB}" "${THRIFT_COMPILER}"
CMAKE_ARGS ${THRIFT_CMAKE_ARGS}
- STEP_TARGETS flex_step libevent_step)
+ STEP_TARGETS flex_step libevent_step
+ DEPENDS ${THRIFT_DEPENDENCIES})
else()
ExternalProject_Add(thrift_ep
URL "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz"
CMAKE_ARGS ${THRIFT_CMAKE_ARGS}
- STEP_TARGETS flex_step libevent_step)
+ STEP_TARGETS flex_step libevent_step
+ DEPENDS ${THRIFT_DEPENDENCIES})
endif()
set(THRIFT_VENDORED 1)
else()
@@ -235,27 +247,28 @@
# Download and configure Windows build of Flex and Bison
ExternalProject_Add_Step(thrift_ep flex_step
- COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/dist/winflexbison &&
- cd thirdparty/dist/winflexbison &&
- curl -SLO https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip &&
- ${CMAKE_COMMAND} -E tar xzf win_flex_bison-${WINFLEXBISON_VERSION}.zip
+ COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/dist/winflexbison
+ COMMAND cd thirdparty/dist/winflexbison
+ COMMAND curl -SLO https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip
+ COMMAND ${CMAKE_COMMAND} -E tar xzf win_flex_bison-${WINFLEXBISON_VERSION}.zip
DEPENDERS configure
+ DEPENDEES download
WORKING_DIRECTORY ${SOURCE_DIR})
# Download and build libevent
ExternalProject_Add_Step(thrift_ep libevent_step
- COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/src &&
- cd thirdparty/src &&
- curl -SLO https://github.com/nmathewson/Libevent/archive/release-${LIBEVENT_VERSION}-rc.zip &&
- ${CMAKE_COMMAND} -E tar xzf release-${LIBEVENT_VERSION}-rc.zip &&
- cd Libevent-release-${LIBEVENT_VERSION}-rc &&
- nmake -f Makefile.nmake &&
- ${CMAKE_COMMAND} -E make_directory lib &&
- copy *.lib lib &&
- xcopy /E /I /D WIN32-Code\\nmake include\\event2 &&
- xcopy /E /I /D WIN32-Code\\nmake\\event2 include\\event2 &&
- copy *.h include
+ COMMAND ${CMAKE_COMMAND} -E make_directory thirdparty/src
+ COMMAND cd thirdparty/src
+ COMMAND curl -SLO https://github.com/nmathewson/Libevent/archive/release-${LIBEVENT_VERSION}-rc.zip
+ COMMAND ${CMAKE_COMMAND} -E tar xzf release-${LIBEVENT_VERSION}-rc.zip
+ COMMAND cd Libevent-release-${LIBEVENT_VERSION}-rc
+ COMMAND ${CMAKE_COMMAND} -E make_directory build
+ COMMAND cd build
+ COMMAND ${CMAKE_COMMAND} -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=.. ..
+ COMMAND nmake
+ COMMAND nmake install
DEPENDERS configure
+ DEPENDEES download
WORKING_DIRECTORY ${SOURCE_DIR})
endif()
@@ -554,13 +567,16 @@
-DARROW_IPC=OFF
-DARROW_BOOST_USE_SHARED=${PARQUET_BOOST_USE_SHARED}
-DARROW_BUILD_TESTS=OFF)
-
+
if (MSVC)
- set(ARROW_CMAKE_ARGS -G "NMake Makefiles" ${ARROW_CMAKE_ARGS})
+ set(ARROW_CMAKE_ARGS -G "NMake Makefiles" -DARROW_CXXFLAGS="/WX" ${ARROW_CMAKE_ARGS})
+ if (NOT PARQUET_BOOST_USE_SHARED)
+ set(ARROW_CMAKE_ARGS -DARROW_BOOST_USE_SHARED=OFF ${ARROW_CMAKE_ARGS})
+ endif()
endif()
if ("$ENV{PARQUET_ARROW_VERSION}" STREQUAL "")
- set(ARROW_VERSION "f7ab7270bb07466dabf84c015a6db2a192eb3dad")
+ set(ARROW_VERSION "2c3e111d45c056d429cef312533c9f3f96b08ae8")
else()
set(ARROW_VERSION "$ENV{PARQUET_ARROW_VERSION}")
endif()
@@ -581,7 +597,7 @@
CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp
CMAKE_GENERATOR "NMake Makefiles"
CMAKE_GENERATOR_PLATFORM "x64"
- BUILD_COMMAND nmake && nmake install
+ BUILD_COMMAND nmake COMMAND nmake install
STEP_TARGETS copy_dll_step)
else()
ExternalProject_Add(arrow_ep
@@ -589,12 +605,13 @@
CONFIGURE_COMMAND "${CMAKE_COMMAND}" ${ARROW_CMAKE_ARGS} ${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp
CMAKE_GENERATOR "NMake Makefiles"
CMAKE_GENERATOR_PLATFORM "x64"
- BUILD_COMMAND nmake && nmake install
+ BUILD_COMMAND nmake COMMAND nmake install
STEP_TARGETS copy_dll_step)
endif()
ExternalProject_Get_Property(arrow_ep SOURCE_DIR)
ExternalProject_Add_Step(arrow_ep copy_dll_step
+ COMMAND ${CMAKE_COMMAND} -E make_directory ${BUILD_OUTPUT_ROOT_DIRECTORY}
COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_SHARED_LIB} ${BUILD_OUTPUT_ROOT_DIRECTORY}
DEPENDEES build
WORKING_DIRECTORY ${SOURCE_DIR})
diff --git a/src/parquet/arrow/arrow-reader-writer-test.cc b/src/parquet/arrow/arrow-reader-writer-test.cc
index 48790d9..64690ae 100644
--- a/src/parquet/arrow/arrow-reader-writer-test.cc
+++ b/src/parquet/arrow/arrow-reader-writer-test.cc
@@ -661,10 +661,10 @@
// 2nd January 1970, 11:35min 145738543ns
Int96 day;
- day.value[2] = 2440589l;
+ day.value[2] = UINT32_C(2440589);
int64_t seconds = (11 * 60 + 35) * 60;
*(reinterpret_cast<int64_t*>(&(day.value))) =
- seconds * 1000l * 1000l * 1000l + 145738543;
+ seconds * INT64_C(1000) * INT64_C(1000) * INT64_C(1000) + 145738543;
// Compute the corresponding nanosecond timestamp
struct tm datetime = {0};
datetime.tm_year = 70;
@@ -676,7 +676,7 @@
epoch.tm_year = 70;
epoch.tm_mday = 1;
// Nanoseconds since the epoch
- int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * 1000000000;
+ int64_t val = lrint(difftime(mktime(&datetime), mktime(&epoch))) * INT64_C(1000000000);
val += 145738543;
std::vector<std::shared_ptr<schema::Node>> fields(
@@ -1064,7 +1064,7 @@
TEST(TestArrowWrite, CheckChunkSize) {
const int num_columns = 2;
const int num_rows = 128;
- const int64_t chunk_size = 0; // note the chunk_size is 0
+ const int64_t chunk_size = 0; // note the chunk_size is 0
std::shared_ptr<Table> table;
MakeDoubleTable(num_columns, num_rows, 1, &table);
diff --git a/src/parquet/util/bit-util.h b/src/parquet/util/bit-util.h
index 8f0a270..559d5d5 100644
--- a/src/parquet/util/bit-util.h
+++ b/src/parquet/util/bit-util.h
@@ -329,7 +329,7 @@
// Returns the minimum number of bits needed to represent the value of 'x'
static inline int NumRequiredBits(uint64_t x) {
for (int i = 63; i >= 0; --i) {
- if (x & 1L << i) return i + 1;
+ if (x & (UINT64_C(1) << i)) return i + 1;
}
return 0;
}
