.github/workflows/ci.yml - drill - Git at Google


            
            #

            
            # Licensed to the Apache Software Foundation (ASF) under one

            
            # or more contributor license agreements.  See the NOTICE file

            
            # distributed with this work for additional information

            
            # regarding copyright ownership.  The ASF licenses this file

            
            # to you under the Apache License, Version 2.0 (the

            
            # "License"); you may not use this file except in compliance

            
            # with the License.  You may obtain a copy of the License at

            
            #

            
            # http://www.apache.org/licenses/LICENSE-2.0

            
            #

            
            # Unless required by applicable law or agreed to in writing, software

            
            # distributed under the License is distributed on an "AS IS" BASIS,

            
            # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

            
            # See the License for the specific language governing permissions and

            
            # limitations under the License.

            
            #


            # This workflow is triggered for each push event to the master branch or pull request.

            
            # It contains two jobs:

            
            # - "Main Build" job runs all unit tests jor two LTS java versions and the latest Java version.

            
            # - "Run checkstyle and generate protobufs" job builds Drill and checks its license,

            
            # builds Drill native client and regenerates protobufs to ensure that committed files are up-to-date.

            
            name: Github CI


            on: [push, pull_request]


            jobs:

            
              build:

            
                name: Main Build (java ${{ matrix.java }}, shard ${{ matrix.shard }})

            
                runs-on: ubuntu-latest

            
                if: github.repository == 'apache/drill'

            
                timeout-minutes: 150

            
                strategy:

            
                  matrix:

            
                    # Java versions to run unit tests (Jetty 12 requires Java 17+)

            
                    java: [ '17', '21', '25' ]

            
                    profile: ['default-hadoop']

            
                    # exec/java-exec holds ~31 minutes of the suite in 577 classes, which dominates

            
                    # the build. Splitting it three ways by class name brings the longest shard to

            
                    # about 11 minutes. Boundaries were chosen from measured per-class times; the

            
                    # patterns are exhaustive, so a newly added test always lands in exactly one

            
                    # shard rather than being silently skipped.

            
                    shard: [ '1', '2', '3' ]

            
                  fail-fast: false

            
                steps:

            
                  - name: Checkout

            
                    uses: actions/checkout@v7

            
                  - name: Setup java

            
                    uses: actions/setup-java@v6

            
                    with:

            
                      distribution: 'temurin'

            
                      java-version: ${{ matrix.java }}

            
                      cache: 'maven'

            
                  - name: Set up swap space

            
                    # Linux Action Runners come with 7GB of RAM which isn't quite enoough

            
                    # to run the test suite. Two workarounds are used below: a swap file is

            
                    # added to the Runner and memory hungry tests are run separately.

            
                    # dd rather than fallocate: swapon rejects fallocated files on some runner

            
                    # filesystems ("Invalid argument"). A missing swap file only costs headroom,

            
                    # so warn instead of failing the job.

            
                    run: |

            
                      sudo sh -c "

            
                        dd if=/dev/zero of=/drill-swapfile bs=1M count=2048 status=none &&

            
                        chmod 0600 /drill-swapfile &&

            
                        mkswap /drill-swapfile &&

            
                        swapon /drill-swapfile

            
                      " || echo "::warning::Could not add swap space; continuing without it"

            
                  # Which slice of the suite this job runs. The patterns are spelled out one prefix

            
                  # at a time because surefire does NOT support character-class globs:

            
                  # -Dtest='Test[A-F]*' silently matches nothing and the build passes green having

            
                  # run no tests. Only '*' wildcards and comma separated lists work, so each letter

            
                  # gets its own entry, in both cases (TestforBaseTestInheritance has a lowercase f).

            
                  #

            
                  # Shard 3 repeats surefire's other default includes (*Test, *Tests, *TestCase) to

            
                  # pick up classes not named Test*. Verified exhaustive against every test

            
                  # class in the repo. TestFrameworkTest matches shards 1 and 3 and runs twice —

            
                  # harmless, and far safer than a gap that skips tests without saying so.

            
                  - name: Select test shard

            
                    id: shard

            
                    run: |

            
                      case "${{ matrix.shard }}" in

            
                        1) echo 'pattern=TestA*,Testa*,TestB*,Testb*,TestC*,Testc*,TestD*,Testd*,TestE*,Teste*,TestF*,Testf*' >> "$GITHUB_OUTPUT" ;;

            
                        2) echo 'pattern=TestG*,Testg*,TestH*,Testh*,TestI*,Testi*,TestJ*,Testj*,TestK*,Testk*,TestL*,Testl*,TestM*,Testm*,TestN*,Testn*,TestO*,Testo*,TestP*,Testp*' >> "$GITHUB_OUTPUT" ;;

            
                        3) echo 'pattern=TestQ*,Testq*,TestR*,Testr*,TestS*,Tests*,TestT*,Testt*,TestU*,Testu*,TestV*,Testv*,TestW*,Testw*,TestX*,Testx*,TestY*,Testy*,TestZ*,Testz*,*Test,*Tests,*TestCase,Test0*,Test1*,Test2*,Test3*,Test4*,Test5*,Test6*,Test7*,Test8*,Test9*,Test_*' >> "$GITHUB_OUTPUT" ;;

            
                        *) echo "::error::Unknown shard '${{ matrix.shard }}'; expected 1, 2 or 3"; exit 1 ;;

            
                      esac


                  # -DforkCount=2: several of the slowest classes are wall-clock bound rather than

            
                  # CPU bound — TestTimedCallable spends 105 seconds asleep, and the RPC, SSL and

            
                  # graceful-shutdown tests wait on timeouts — so a second fork overlaps that idle

            
                  # time at little extra cost. Raise further only with memory headroom: the vector

            
                  # tests fail with OutOfMemoryException under pressure.

            
                  - name: Build and test

            
                    run: |

            
                      mvn -P${{ matrix.profile }} install --batch-mode --no-transfer-progress \

            
                      -DforkCount=2 \

            
                      -Dsurefire.failIfNoSpecifiedTests=false \

            
                      -Dtest='${{ steps.shard.outputs.pattern }}' \

            
                      -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120

            
                  - name: Remove swap space

            
                    run : |

            
                      free -h

            
                      sudo swapoff /drill-swapfile 2>/dev/null || true

            
                      sudo rm -f /drill-swapfile

            
              checkstyle_protobuf:

            
                name: Run checkstyle and generate protobufs

            
                runs-on: ubuntu-22.04

            
                if: github.repository == 'apache/drill'

            
                steps:

            
                  - name: Checkout

            
                    uses: actions/checkout@v7

            
                  - name: Setup java

            
                    uses: actions/setup-java@v6

            
                    with:

            
                      distribution: 'temurin'

            
                      java-version: '17'

            
                      cache: 'maven'

            
                  # Caches built protobuf library

            
                  - name: Cache protobufs

            
                    uses: actions/cache@v6

            
                    with:

            
                      path: ~/protobuf

            
                      key: ${{ runner.os }}-protobuf

            
                  # Install libraries required for protobuf generation

            
                  - name: Install dependencies

            
                    run: |

            
                      sudo apt update -y && sudo apt install -y libboost-all-dev libzookeeper-mt-dev libsasl2-dev cmake libcppunit-dev checkinstall && \

            
                      pushd .. && \

            
                      if [ -f $HOME/protobuf/protobuf_3.16.3* ]; then \

            
                        sudo dpkg -i $HOME/protobuf/protobuf_3.16.3*; \

            
                      else \

            
                        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.16.3/protobuf-java-3.16.3.zip && \

            
                        unzip protobuf-java-3.16.3.zip && pushd protobuf-3.16.3 && \

            
                        ./configure && sudo make && sudo checkinstall -y && \

            
                        if [ ! -d $HOME/protobuf ]; then \

            
                          mkdir -p $HOME/protobuf; \

            
                        fi && \

            
                        mv protobuf_3.16.3* $HOME/protobuf/ && popd; \

            
                      fi && \

            
                      sudo ldconfig && popd; \

            
                  # Builds Drill project, performs license checkstyle goal and regenerates java and C++ protobuf files

            
                  - name: Build

            
                    run: |

            
                      MAVEN_OPTS="-Xms1G -Xmx1G" mvn install -Drat.skip=false -Dlicense.skip=false --batch-mode --no-transfer-progress -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true && \

            
                      pushd protocol && mvn process-sources -P proto-compile && popd && \

            
                      mkdir contrib/native/client/build && pushd contrib/native/client/build && cmake -G "Unix Makefiles" .. && make cpProtobufs && popd; \

            
                  # Checks whether project files weren't changed after regenerating protobufs

            
                  - name: Check protobufs

            
                    run: |

            
                      if [ "$(git status -s | grep -c "")" -gt 0 ]; then \

            
                        echo "The following changes are found in files after regenerating protobufs (output may be used as a patchto apply):" >&2 &&

            
                        echo "$(git diff --color)" && \

            
                        exit 1; \

            
                      else

            
                        echo "All checks are passed!";

            
                      fi
	#
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	#

	# This workflow is triggered for each push event to the master branch or pull request.
	# It contains two jobs:
	# - "Main Build" job runs all unit tests jor two LTS java versions and the latest Java version.
	# - "Run checkstyle and generate protobufs" job builds Drill and checks its license,
	# builds Drill native client and regenerates protobufs to ensure that committed files are up-to-date.
	name: Github CI

	on: [push, pull_request]

	jobs:
	build:
	name: Main Build (java ${{ matrix.java }}, shard ${{ matrix.shard }})
	runs-on: ubuntu-latest
	if: github.repository == 'apache/drill'
	timeout-minutes: 150
	strategy:
	matrix:
	# Java versions to run unit tests (Jetty 12 requires Java 17+)
	java: [ '17', '21', '25' ]
	profile: ['default-hadoop']
	# exec/java-exec holds ~31 minutes of the suite in 577 classes, which dominates
	# the build. Splitting it three ways by class name brings the longest shard to
	# about 11 minutes. Boundaries were chosen from measured per-class times; the
	# patterns are exhaustive, so a newly added test always lands in exactly one
	# shard rather than being silently skipped.
	shard: [ '1', '2', '3' ]
	fail-fast: false
	steps:
	- name: Checkout
	uses: actions/checkout@v7
	- name: Setup java
	uses: actions/setup-java@v6
	with:
	distribution: 'temurin'
	java-version: ${{ matrix.java }}
	cache: 'maven'
	- name: Set up swap space
	# Linux Action Runners come with 7GB of RAM which isn't quite enoough
	# to run the test suite. Two workarounds are used below: a swap file is
	# added to the Runner and memory hungry tests are run separately.
	# dd rather than fallocate: swapon rejects fallocated files on some runner
	# filesystems ("Invalid argument"). A missing swap file only costs headroom,
	# so warn instead of failing the job.
	run: \|
	sudo sh -c "
	dd if=/dev/zero of=/drill-swapfile bs=1M count=2048 status=none &&
	chmod 0600 /drill-swapfile &&
	mkswap /drill-swapfile &&
	swapon /drill-swapfile
	" \|\| echo "::warning::Could not add swap space; continuing without it"
	# Which slice of the suite this job runs. The patterns are spelled out one prefix
	# at a time because surefire does NOT support character-class globs:
	# -Dtest='Test[A-F]*' silently matches nothing and the build passes green having
	# run no tests. Only '*' wildcards and comma separated lists work, so each letter
	# gets its own entry, in both cases (TestforBaseTestInheritance has a lowercase f).
	#
	# Shard 3 repeats surefire's other default includes (Test, Tests, *TestCase) to
	# pick up classes not named Test*. Verified exhaustive against every test
	# class in the repo. TestFrameworkTest matches shards 1 and 3 and runs twice —
	# harmless, and far safer than a gap that skips tests without saying so.
	- name: Select test shard
	id: shard
	run: \|
	case "${{ matrix.shard }}" in
	1) echo 'pattern=TestA,Testa,TestB,Testb,TestC,Testc,TestD,Testd,TestE,Teste,TestF,Testf' >> "$GITHUB_OUTPUT" ;;
	2) echo 'pattern=TestG,Testg,TestH,Testh,TestI,Testi,TestJ,Testj,TestK,Testk,TestL,Testl,TestM,Testm,TestN,Testn,TestO,Testo,TestP,Testp' >> "$GITHUB_OUTPUT" ;;
	3) echo 'pattern=TestQ,Testq,TestR,Testr,TestS,Tests,TestT,Testt,TestU,Testu,TestV,Testv,TestW,Testw,TestX,Testx,TestY,Testy,TestZ,Testz,Test,Tests,TestCase,Test0,Test1,Test2,Test3,Test4,Test5,Test6,Test7,Test8,Test9,Test_' >> "$GITHUB_OUTPUT" ;;
	*) echo "::error::Unknown shard '${{ matrix.shard }}'; expected 1, 2 or 3"; exit 1 ;;
	esac

	# -DforkCount=2: several of the slowest classes are wall-clock bound rather than
	# CPU bound — TestTimedCallable spends 105 seconds asleep, and the RPC, SSL and
	# graceful-shutdown tests wait on timeouts — so a second fork overlaps that idle
	# time at little extra cost. Raise further only with memory headroom: the vector
	# tests fail with OutOfMemoryException under pressure.
	- name: Build and test
	run: \|
	mvn -P${{ matrix.profile }} install --batch-mode --no-transfer-progress \
	-DforkCount=2 \
	-Dsurefire.failIfNoSpecifiedTests=false \
	-Dtest='${{ steps.shard.outputs.pattern }}' \
	-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
	- name: Remove swap space
	run : \|
	free -h
	sudo swapoff /drill-swapfile 2>/dev/null \|\| true
	sudo rm -f /drill-swapfile
	checkstyle_protobuf:
	name: Run checkstyle and generate protobufs
	runs-on: ubuntu-22.04
	if: github.repository == 'apache/drill'
	steps:
	- name: Checkout
	uses: actions/checkout@v7
	- name: Setup java
	uses: actions/setup-java@v6
	with:
	distribution: 'temurin'
	java-version: '17'
	cache: 'maven'
	# Caches built protobuf library
	- name: Cache protobufs
	uses: actions/cache@v6
	with:
	path: ~/protobuf
	key: ${{ runner.os }}-protobuf
	# Install libraries required for protobuf generation
	- name: Install dependencies
	run: \|
	sudo apt update -y && sudo apt install -y libboost-all-dev libzookeeper-mt-dev libsasl2-dev cmake libcppunit-dev checkinstall && \
	pushd .. && \
	if [ -f $HOME/protobuf/protobuf_3.16.3* ]; then \
	sudo dpkg -i $HOME/protobuf/protobuf_3.16.3*; \
	else \
	wget https://github.com/protocolbuffers/protobuf/releases/download/v3.16.3/protobuf-java-3.16.3.zip && \
	unzip protobuf-java-3.16.3.zip && pushd protobuf-3.16.3 && \
	./configure && sudo make && sudo checkinstall -y && \
	if [ ! -d $HOME/protobuf ]; then \
	mkdir -p $HOME/protobuf; \
	fi && \
	mv protobuf_3.16.3* $HOME/protobuf/ && popd; \
	fi && \
	sudo ldconfig && popd; \
	# Builds Drill project, performs license checkstyle goal and regenerates java and C++ protobuf files
	- name: Build
	run: \|
	MAVEN_OPTS="-Xms1G -Xmx1G" mvn install -Drat.skip=false -Dlicense.skip=false --batch-mode --no-transfer-progress -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true && \
	pushd protocol && mvn process-sources -P proto-compile && popd && \
	mkdir contrib/native/client/build && pushd contrib/native/client/build && cmake -G "Unix Makefiles" .. && make cpProtobufs && popd; \
	# Checks whether project files weren't changed after regenerating protobufs
	- name: Check protobufs
	run: \|
	if [ "$(git status -s \| grep -c "")" -gt 0 ]; then \
	echo "The following changes are found in files after regenerating protobufs (output may be used as a patchto apply):" >&2 &&
	echo "$(git diff --color)" && \
	exit 1; \
	else
	echo "All checks are passed!";
	fi