| name: "MiNiFi-CPP CI" |
| on: [push, pull_request, workflow_dispatch] |
| env: |
| DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ |
| -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ |
| -DENABLE_ELASTICSEARCH=OFF -DDOCKER_BUILD_ONLY=ON |
| SCCACHE_GHA_ENABLE: true |
| jobs: |
| macos_xcode: |
| name: "macos-xcode" |
| runs-on: macos-latest-xlarge |
| timeout-minutes: 180 |
| env: |
| CCACHE_BASEDIR: ${{ GITHUB.WORKSPACE }} |
| CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache |
| steps: |
| - id: checkout |
| uses: actions/checkout@v3 |
| - name: cache restore |
| uses: actions/cache/restore@v3 |
| with: |
| path: ${{ env.CCACHE_DIR }} |
| key: macos-xcode-ccache-${{github.ref}}-${{github.sha}} |
| restore-keys: | |
| macos-xcode-ccache-${{github.ref}}- |
| macos-xcode-ccache-refs/heads/main- |
| - id: install_dependencies |
| run: | |
| # Skip brew update until https://github.com/actions/setup-python/issues/577 is fixed |
| # brew update |
| HOMEBREW_NO_AUTO_UPDATE=1 brew install ossp-uuid flex lua ccache sqliteodbc automake autoconf ninja |
| - id: setup_env |
| name: setup enviroment |
| run: | |
| echo "PATH=/usr/lib/ccache:/usr/local/opt/ccache/bin:/usr/local/opt/ccache/libexec:$PATH" >> $GITHUB_ENV |
| echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null |
| - name: build |
| run: | |
| export PATH="/usr/local/opt/flex/bin:$PATH" |
| export LDFLAGS="-L/usr/local/opt/flex/lib" |
| export CPPFLAGS="-I/usr/local/opt/flex/include" |
| # CPPFLAGS are not recognized by cmake, so we have to force them to CFLAGS and CXXFLAGS to have flex 2.6 working |
| ./bootstrap.sh -e -t && cd build && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DCMAKE_C_FLAGS="${CPPFLAGS} ${CFLAGS}" -DCMAKE_CXX_FLAGS="${CPPFLAGS} ${CXXFLAGS}" -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_SQL=ON -DUSE_REAL_ODBC_TEST_DRIVER=ON -DENABLE_AZURE=ON -DENABLE_GCP=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES=OFF -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON .. && cmake --build . --parallel 4 |
| - name: cache save |
| uses: actions/cache/save@v3 |
| if: always() |
| with: |
| path: ${{ env.CCACHE_DIR }} |
| key: macos-xcode-ccache-${{github.ref}}-${{github.sha}} |
| - name: test |
| id: test |
| run: | |
| # Set core file size limit to 1GiB |
| ulimit -c 1048576 |
| cd build && ctest -j4 --output-on-failure --timeout 300 |
| - name: linter |
| run: cd build && ninja linter |
| - name: check-cores |
| if: ${{ failure() && steps.test.conclusion == 'failure' }} |
| run: | |
| if [ "$(ls -A /cores)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi |
| - uses: actions/upload-artifact@v3.1.2 |
| if: ${{ failure() && env.CORES_EXIST == 'true' }} |
| with: |
| name: macos-coredumps |
| path: /cores |
| - uses: actions/upload-artifact@v3.1.2 |
| if: ${{ failure() && env.CORES_EXIST == 'true' }} |
| with: |
| name: macos-binaries |
| path: build/bin |
| windows_VS2019: |
| name: "windows-2019" |
| runs-on: windows-2019 |
| timeout-minutes: 180 |
| steps: |
| - name: Support longpaths |
| run: git config --system core.longpaths true |
| - id: checkout |
| uses: actions/checkout@v3 |
| - name: sccache cache restore |
| uses: actions/cache/restore@v3 |
| with: |
| path: ~/AppData/Local/Mozilla/sccache/cache |
| key: ${{ runner.os }}-sccache-${{ github.ref }}-${{ github.sha }} |
| restore-keys: | |
| ${{ runner.os }}-sccache-${{ github.ref }} |
| ${{ runner.os }}-sccache-refs/heads/main |
| ${{ runner.os }}-sccache |
| - name: Run sccache-cache |
| uses: mozilla-actions/sccache-action@v0.0.3 |
| - name: Install ninja-build tool |
| uses: seanmiddleditch/gha-setup-ninja@v3 |
| - name: Set up Python |
| uses: actions/setup-python@v4 |
| with: |
| python-version: '3.11' |
| - name: Set up Lua |
| uses: xpol/setup-lua@v0.3 |
| - name: Set up NASM for OpenSSL |
| uses: ilammy/setup-nasm@v1 |
| - id: install-sqliteodbc-driver |
| run: | |
| Invoke-WebRequest -Uri "http://www.ch-werner.de/sqliteodbc/sqliteodbc_w64.exe" -OutFile "sqliteodbc_w64.exe" |
| if ((Get-FileHash 'sqliteodbc_w64.exe').Hash -ne "0df79be4a4412542839ebf405b20d95a7dfc803da0b0b6b0dc653d30dc82ee84") {Write "Hash mismatch"; Exit 1} |
| ./sqliteodbc_w64.exe /S |
| shell: powershell |
| - name: build |
| run: | |
| for /f "usebackq delims=" %%i in (`vswhere.exe -latest -property installationPath`) do if exist "%%i\Common7\Tools\vsdevcmd.bat" call "%%i\Common7\Tools\vsdevcmd.bat" -arch=x64 -host_arch=x64 |
| win_build_vs.bat ..\b /64 /CI /S /A /PDH /SPLUNK /GCP /ELASTIC /K /L /R /Z /N /RO /PR /PYTHON_SCRIPTING /LUA_SCRIPTING /MQTT /SCCACHE /NINJA |
| sccache --show-stats |
| shell: cmd |
| - name: sccache cache save |
| uses: actions/cache/save@v3 |
| if: always() |
| with: |
| path: ~/AppData/Local/Mozilla/sccache/cache |
| key: ${{ runner.os }}-sccache-${{ github.ref }}-${{ github.sha }} |
| - name: test |
| run: cd ..\b && ctest --timeout 300 --parallel %NUMBER_OF_PROCESSORS% -C Release --output-on-failure |
| shell: cmd |
| - name: linter |
| run: cd ..\b && cmake --build . --target linter --config Release -j 8 |
| shell: cmd |
| ubuntu_20_04: |
| name: "ubuntu-20.04" |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 120 |
| steps: |
| - id: checkout |
| uses: actions/checkout@v3 |
| - name: cache restore |
| uses: actions/cache/restore@v3 |
| with: |
| path: ~/.ccache |
| key: ubuntu-20.04-ccache-${{github.ref}}-${{github.sha}} |
| restore-keys: | |
| ubuntu-20.04-ccache-${{github.ref}}- |
| ubuntu-20.04-ccache-refs/heads/main- |
| - id: install_deps |
| run: | |
| sudo apt update |
| sudo apt install -y ccache libfl-dev libpcap-dev |
| echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV |
| echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null |
| - name: build |
| run: | |
| export CC=gcc-11 |
| export CXX=g++-11 |
| ./bootstrap.sh -e -t |
| cd build |
| cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_SQL=OFF -DENABLE_LIBRDKAFKA=OFF -DENABLE_AWS=OFF \ |
| -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON -DENABLE_JNI=ON -DENABLE_SFTP=ON \ |
| -DENABLE_LUA_SCRIPTING=OFF -DENABLE_PYTHON_SCRIPTING=OFF -DENABLE_MQTT=OFF -DENABLE_ELASTICSEARCH=OFF -DENABLE_KUBERNETES=OFF -DENABLE_OPC=OFF .. |
| make -j$(nproc) VERBOSE=1 |
| - name: cache save |
| uses: actions/cache/save@v3 |
| if: always() |
| with: |
| path: ~/.ccache |
| key: ubuntu-20.04-ccache-${{github.ref}}-${{github.sha}} |
| - name: test |
| id: test |
| run: | |
| # Set core file size limit to 1GiB |
| ulimit -c 1048576 |
| cd build && make test ARGS="--timeout 300 -j8 --output-on-failure" |
| - name: check-cores |
| if: ${{ failure() && steps.test.conclusion == 'failure' }} |
| run: | |
| if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi |
| - uses: actions/upload-artifact@v3.1.2 |
| if: ${{ failure() && env.CORES_EXIST == 'true' }} |
| with: |
| name: ubuntu-coredumps |
| path: /var/lib/apport/coredump/ |
| - uses: actions/upload-artifact@v3.1.2 |
| if: ${{ failure() && env.CORES_EXIST == 'true' }} |
| with: |
| name: ubuntu-binaries |
| path: build/bin |
| ubuntu_22_04_clang: |
| name: "ubuntu-22.04-clang" |
| runs-on: ubuntu-22.04 |
| timeout-minutes: 240 |
| steps: |
| - id: checkout |
| uses: actions/checkout@v3 |
| - name: cache restore |
| uses: actions/cache/restore@v3 |
| with: |
| path: ~/.ccache |
| key: ubuntu-22.04-all-clang-ccache-${{github.ref}}-${{github.sha}} |
| restore-keys: | |
| ubuntu-22.04-all-clang-ccache-${{github.ref}}- |
| ubuntu-22.04-all-clang-ccache-refs/heads/main- |
| - id: install_deps |
| run: | |
| wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add - |
| echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main" | sudo tee -a /etc/apt/sources.list |
| echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy main" | sudo tee -a /etc/apt/sources.list |
| echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main" | sudo tee -a /etc/apt/sources.list |
| echo "deb-src http://apt.llvm.org/jammy/ llvm-toolchain-jammy-16 main" | sudo tee -a /etc/apt/sources.list |
| sudo apt update |
| sudo apt install -y ccache libfl-dev libpcap-dev libusb-1.0-0-dev libpng-dev libgps-dev clang-16 clang-tidy-16 libc++-16-dev libc++abi-16-dev libc++1-16 libc++abi1-16 libunwind-16 libsqliteodbc lua5.3 liblua5.3-dev flake8 parallel |
| echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV |
| echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null |
| - id: free_disk_space |
| run: | |
| # We can gain additional disk space on the Ubuntu runners thanks to these suggestions: |
| # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 |
| # https://github.com/actions/runner-images/issues/2606#issuecomment-772683150 |
| sudo rm -rf /usr/share/dotnet |
| sudo rm -rf /usr/local/lib/android |
| sudo rm -rf /opt/ghc |
| sudo rm -rf "/usr/local/share/boost" |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" |
| - name: build |
| run: | |
| export CC=clang-16 |
| export CXX=clang++-16 |
| ./bootstrap.sh -e -t |
| cd build |
| export CXXFLAGS="${CXXFLAGS} -stdlib=libc++" |
| export LDFLAGS="${LDFLAGS} -stdlib=libc++" |
| cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \ |
| -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \ |
| -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \ |
| -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \ |
| -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. |
| cmake --build . --parallel $(nproc) |
| - id: cache_save |
| uses: actions/cache/save@v3 |
| if: always() |
| with: |
| path: ~/.ccache |
| key: ubuntu-22.04-all-clang-ccache-${{github.ref}}-${{github.sha}} |
| - name: test |
| id: test |
| run: | |
| # Set core file size limit to 1GiB |
| ulimit -c 1048576 |
| cd build && make test ARGS="--timeout 300 -j8 --output-on-failure" |
| - name: linter |
| run: cd build && make -j$(nproc) linter |
| - name: shellcheck |
| run: cd build && make shellcheck |
| - id: flake8_check |
| run: cd build && make flake8 |
| - id: files |
| uses: Ana06/get-changed-files@v2.2.0 |
| continue-on-error: true |
| - name: clang-tidy |
| run: | |
| if [[ -n "${{ steps.files.outputs.all }}" ]]; then |
| FILES="${{ steps.files.outputs.all }}" |
| else |
| git fetch origin main && git checkout -b main origin/main || true |
| FILES=$(git diff --name-only main ${{ github.sha }} --) |
| git checkout ${{ github.sha }} |
| fi |
| # https://stackoverflow.com/questions/58466701/clang-tidy-cant-locate-stdlib-headers |
| sed -i -e 's/\/usr\/lib\/ccache\/clang++-16/\/lib\/llvm-16\/bin\/clang++/g' build/compile_commands.json |
| sed -i -e 's/\/usr\/lib\/ccache\/clang-16/\/lib\/llvm-16\/bin\/clang/g' build/compile_commands.json |
| sed -i -e 's/\/usr\/lib\/ccache\/c++/\/lib\/llvm-16\/bin\/clang++/g' build/compile_commands.json |
| sed -i -e 's/\/usr\/lib\/ccache\/cc/\/lib\/llvm-16\/bin\/clang/g' build/compile_commands.json |
| |
| parallel -j$(( $(nproc) + 1 )) ./run_clang_tidy.sh ::: ${FILES} |
| - name: check-cores |
| if: ${{ failure() && steps.test.conclusion == 'failure' }} |
| run: | |
| if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi |
| - uses: actions/upload-artifact@v3.1.2 |
| if: ${{ failure() && env.CORES_EXIST == 'true' }} |
| with: |
| name: ubuntu-clang-coredumps |
| path: /var/lib/apport/coredump/ |
| - uses: actions/upload-artifact@v3.1.2 |
| if: ${{ failure() && env.CORES_EXIST == 'true' }} |
| with: |
| name: ubuntu-clang-binaries |
| path: build/bin |
| centos: |
| name: "centos" |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 180 |
| steps: |
| - id: checkout |
| uses: actions/checkout@v3 |
| - name: cache restore |
| uses: actions/cache/restore@v3 |
| with: |
| path: ~/.ccache |
| key: centos-ccache-${{github.ref}}-${{github.sha}} |
| restore-keys: | |
| centos-ccache-${{github.ref}}- |
| centos-ccache-refs/heads/main- |
| - id: install_deps |
| run: | |
| sudo apt update |
| sudo apt install -y ccache |
| echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV |
| - id: free_disk_space |
| run: | |
| # We can gain additional disk space on the Ubuntu runners thanks to these suggestions: |
| # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 |
| # https://github.com/actions/runner-images/issues/2606#issuecomment-772683150 |
| sudo rm -rf /usr/share/dotnet |
| sudo rm -rf /usr/local/lib/android |
| sudo rm -rf /opt/ghc |
| sudo rm -rf "/usr/local/share/boost" |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" |
| - id: build |
| run: | |
| if [ -d ~/.ccache ]; then mv ~/.ccache .; fi |
| # centos build can run out of the github runners' disk space if built with RelWithDebInfo so we keep the Release build here |
| mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \ |
| -DENABLE_COAP=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON \ |
| -DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \ |
| -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \ |
| -DENABLE_ELASTICSEARCH=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos-test |
| - name: cache save |
| uses: actions/cache/save@v3 |
| if: always() |
| with: |
| path: ~/.ccache |
| key: centos-ccache-${{github.ref}}-${{github.sha}} |
| - id: test |
| run: | |
| # Set core file size limit to 1GiB |
| ulimit -c 1048576 |
| echo '/cores/core.%e.%p' | sudo tee /proc/sys/kernel/core_pattern |
| sudo mkdir /cores |
| sudo chmod 777 /cores |
| docker run --name centos-test --init --ulimit core=1048576 --mount type=bind,source=/cores,target=/cores apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"' |
| - name: check-cores |
| if: ${{ failure() && steps.test.conclusion == 'failure' }} |
| run: | |
| if [ "$(ls -A /cores)" ]; then |
| echo "CORES_EXIST=true" >> $GITHUB_ENV; |
| docker cp centos-test:/opt/minifi/build/bin /tmp |
| sudo chmod -R 777 /cores |
| fi |
| - uses: actions/upload-artifact@v3.1.2 |
| if: ${{ failure() && env.CORES_EXIST == 'true' }} |
| with: |
| name: centos-crashes |
| path: /cores |
| - uses: actions/upload-artifact@v3.1.2 |
| if: ${{ failure() && env.CORES_EXIST == 'true' }} |
| with: |
| name: centos-binaries |
| path: /tmp/bin |
| docker_build: |
| name: "Docker build for integration tests" |
| runs-on: ubuntu-22.04 |
| timeout-minutes: 180 |
| steps: |
| - id: checkout |
| uses: actions/checkout@v3 |
| - name: cache restore |
| uses: actions/cache/restore@v3 |
| with: |
| path: ~/.ccache |
| key: docker-ccache-${{github.ref}}-${{github.sha}} |
| restore-keys: | |
| docker-ccache-${{github.ref}}- |
| docker-ccache-refs/heads/main |
| - id: build |
| run: | |
| if [ -d ~/.ccache ]; then mv ~/.ccache .; fi |
| mkdir build |
| cd build |
| cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. |
| make docker |
| - name: cache save |
| uses: actions/cache/save@v3 |
| if: always() |
| with: |
| path: ~/.ccache |
| key: docker-ccache-${{github.ref}}-${{github.sha}} |
| - name: Save docker image |
| run: cd build && docker save -o minifi_docker.tar apacheminificpp:$(grep CMAKE_PROJECT_VERSION:STATIC CMakeCache.txt | cut -d "=" -f2) |
| - name: Upload artifact |
| uses: actions/upload-artifact@v3 |
| with: |
| name: minifi_docker |
| path: build/minifi_docker.tar |
| docker_tests: |
| name: "Docker integration tests" |
| needs: docker_build |
| runs-on: ubuntu-20.04 |
| timeout-minutes: 180 |
| steps: |
| - id: checkout |
| uses: actions/checkout@v3 |
| - id: run_cmake |
| name: Run CMake |
| run: | |
| mkdir build |
| cd build |
| cmake ${DOCKER_CMAKE_FLAGS} .. |
| - name: Download artifact |
| uses: actions/download-artifact@v3 |
| with: |
| name: minifi_docker |
| path: build |
| - name: Load Docker image |
| run: | |
| docker load --input ./build/minifi_docker.tar |
| - id: install_deps |
| name: Install dependencies for Docker Verify |
| run: | |
| sudo apt update |
| sudo apt install -y python3-virtualenv |
| - id: test |
| name: Docker Verify |
| working-directory: ./build |
| run: make docker-verify |
| - name: Test Reporter |
| if: always() |
| uses: phoenix-actions/test-reporting@v12 |
| with: |
| name: Docker integration tests |
| path: build/behavex_output/behave/*.xml |
| reporter: java-junit |
| output-to: 'step-summary' |
| only-summary: 'true' |
| list-tests: 'failed' |
| list-suites: 'failed' |
| - name: Upload artifact |
| if: failure() |
| uses: actions/upload-artifact@v3 |
| with: |
| name: behavex_output |
| path: build/behavex_output |