Fuzz: Wire up ClusterFuzzLite CI fuzzing (#2527)
diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile
new file mode 100644
index 0000000..579c99a
--- /dev/null
+++ b/.clusterfuzzlite/Dockerfile
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM gcr.io/oss-fuzz-base/base-builder-rust
+COPY . $SRC/datafusion-sqlparser-rs
+WORKDIR $SRC/datafusion-sqlparser-rs
+COPY ./.clusterfuzzlite/build.sh $SRC/
diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh
new file mode 100755
index 0000000..a38f944
--- /dev/null
+++ b/.clusterfuzzlite/build.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eu
+
+cd "$SRC/datafusion-sqlparser-rs"
+# the base image's RUSTUP_TOOLCHAIN (nightly) overrides the rust-toolchain pin, so we name none
+cargo fuzz build -O --fuzz-dir fuzz
+
+targets=$(cargo fuzz list --fuzz-dir fuzz)
+if [ -z "$targets" ]; then
+ echo "cargo fuzz list named no target" >&2
+ exit 1
+fi
+
+target_dir=fuzz/target/x86_64-unknown-linux-gnu/release
+for name in $targets; do
+ cp "$target_dir/$name" "$OUT/"
+done
diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml
new file mode 100644
index 0000000..c4b8146
--- /dev/null
+++ b/.clusterfuzzlite/project.yaml
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+language: rust
+sanitizers:
+ - address
diff --git a/.github/workflows/cflite_batch.yml b/.github/workflows/cflite_batch.yml
new file mode 100644
index 0000000..7694ae9
--- /dev/null
+++ b/.github/workflows/cflite_batch.yml
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: ClusterFuzzLite batch fuzzing
+on:
+ schedule:
+ - cron: '0 06 * * *' # 06:00 UTC daily
+permissions: read-all
+jobs:
+ BatchFuzzing:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ strategy:
+ fail-fast: false
+ matrix:
+ sanitizer:
+ - address
+ steps:
+ - name: Build Fuzzers (${{ matrix.sanitizer }})
+ id: build
+ uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+ with:
+ language: rust
+ sanitizer: ${{ matrix.sanitizer }}
+ - name: Run Fuzzers (${{ matrix.sanitizer }})
+ id: run
+ uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ fuzz-seconds: 3600
+ mode: 'batch'
+ sanitizer: ${{ matrix.sanitizer }}
+ storage-repo: ${{ github.server_url }}/${{ github.repository }}.git
+ storage-repo-branch: clusterfuzzlite
diff --git a/.github/workflows/cflite_build.yml b/.github/workflows/cflite_build.yml
new file mode 100644
index 0000000..7afbf07
--- /dev/null
+++ b/.github/workflows/cflite_build.yml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: ClusterFuzzLite continuous builds
+on:
+ push:
+ branches:
+ - main
+permissions: read-all
+jobs:
+ Build:
+ runs-on: ubuntu-latest
+ concurrency:
+ group: ${{ github.workflow }}-${{ matrix.sanitizer }}-${{ github.ref }}
+ cancel-in-progress: true
+ strategy:
+ fail-fast: false
+ matrix:
+ sanitizer:
+ - address
+ steps:
+ - name: Build Fuzzers (${{ matrix.sanitizer }})
+ id: build
+ uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+ with:
+ language: rust
+ sanitizer: ${{ matrix.sanitizer }}
+ upload-build: true
diff --git a/.github/workflows/cflite_pr.yml b/.github/workflows/cflite_pr.yml
new file mode 100644
index 0000000..d6480ae
--- /dev/null
+++ b/.github/workflows/cflite_pr.yml
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: ClusterFuzzLite PR fuzzing
+on:
+ pull_request:
+permissions: read-all
+jobs:
+ PR:
+ runs-on: ubuntu-latest
+ concurrency:
+ group: ${{ github.workflow }}-${{ matrix.sanitizer }}-${{ github.ref }}
+ cancel-in-progress: true
+ strategy:
+ fail-fast: false
+ matrix:
+ sanitizer:
+ - address
+ steps:
+ - name: Build Fuzzers (${{ matrix.sanitizer }})
+ id: build
+ uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+ with:
+ language: rust
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ sanitizer: ${{ matrix.sanitizer }}
+ - name: Run Fuzzers (${{ matrix.sanitizer }})
+ id: run
+ uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ fuzz-seconds: 600
+ mode: 'code-change'
+ sanitizer: ${{ matrix.sanitizer }}
diff --git a/.github/workflows/cflite_prune.yml b/.github/workflows/cflite_prune.yml
new file mode 100644
index 0000000..f125eca
--- /dev/null
+++ b/.github/workflows/cflite_prune.yml
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: ClusterFuzzLite corpus pruning
+on:
+ schedule:
+ - cron: '0 05 * * *' # 05:00 UTC, an hour before batch fuzzing
+permissions: read-all
+jobs:
+ Pruning:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - name: Build Fuzzers
+ id: build
+ uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+ with:
+ language: rust
+ - name: Run Fuzzers
+ id: run
+ uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ fuzz-seconds: 600
+ mode: 'prune'
+ storage-repo: ${{ github.server_url }}/${{ github.repository }}.git
+ storage-repo-branch: clusterfuzzlite
diff --git a/docs/fuzzing.md b/docs/fuzzing.md
index 691481e..3d16551 100644
--- a/docs/fuzzing.md
+++ b/docs/fuzzing.md
@@ -27,4 +27,8 @@
cargo +nightly fuzz run fuzz_parse_sql -- -max_total_time=600
```
+ClusterFuzzLite runs continuous fuzzing. Every pull request fuzzes for 10 minutes in
+`code-change` mode, a daily batch job grows the shared corpus stored on the
+`clusterfuzzlite` branch, and a daily prune compacts it.
+
Crashes land in `artifacts/<target>/` and replay with `cargo fuzz run <target> <crash-file>`.