object_store/.circleci/config.yml - arrow-rs - Git at Google

 ---
 # CI Overview
 # -----------
 #
 # Each night:
 #
 #   A build image is created (ci_image) from `docker/Dockerfile.ci` and is
 #   pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
 #   the CI tasks for the day.
 #
 # Every commit:
 #
 # The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
 #
 # On main if all these checks pass it will then additionally compile in "release" mode and
 # publish a docker image to quay.io/influxdb/iox:$COMMIT_SHA
 #
 # Manual CI Image:
 #
 # It is possible to manually trigger a rebuild of the image used in CI. To do this, navigate to
 # https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=main (overriding the
 # branch name if desired). Then:
 # - Click "Run Pipeline" in the top-right
 # - Expand "Add Parameters"
 # - Add a "boolean" parameter called "ci_image" with the value true
 # - Click "Run Pipeline"
 #
 # If you refresh the page you should see a newly running ci_image workflow
 #

 version: 2.1

 orbs:
   win: circleci/windows@4.1

 commands:
   rust_components:
     description: Verify installed components
     steps:
       - run:
           name: Verify installed components
           command: |
             rustup --version
             rustup show
             cargo fmt --version
             cargo clippy --version

   cache_restore:
     description: Restore Cargo Cache
     steps:
       - restore_cache:
           name: Restoring Cargo Cache
           keys:
             - cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
             - cargo-cache-{{ arch }}-{{ .Branch }}
             - cargo-cache
   cache_save:
     description: Save Cargo Cache
     steps:
       - save_cache:
           name: Save Cargo Cache
           paths:
             - /usr/local/cargo/registry
           key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}

 jobs:
   fmt:
     docker:
       - image: quay.io/influxdb/rust:ci
     environment:
       # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
       CARGO_INCREMENTAL: "0"
       # Disable full debug symbol generation to speed up CI build
       # "1" means line tables only, which is useful for panic tracebacks.
       RUSTFLAGS: "-C debuginfo=1"
       # https://github.com/rust-lang/cargo/issues/10280
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
     steps:
       - checkout
       - rust_components
       - cache_restore
       - run:
           name: Rust fmt
           command: cargo fmt --all -- --check
       - cache_save
   lint:
     docker:
       - image: quay.io/influxdb/rust:ci
     environment:
       # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
       CARGO_INCREMENTAL: "0"
       # Disable full debug symbol generation to speed up CI build
       # "1" means line tables only, which is useful for panic tracebacks.
       RUSTFLAGS: "-C debuginfo=1"
       # https://github.com/rust-lang/cargo/issues/10280
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
     steps:
       - checkout
       - rust_components
       - cache_restore
       - run:
           name: Clippy
           command: cargo clippy --all-targets --all-features --workspace -- -D warnings
       - cache_save
   cargo_audit:
     docker:
       - image: quay.io/influxdb/rust:ci
     environment:
       # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
       CARGO_INCREMENTAL: "0"
       # Disable full debug symbol generation to speed up CI build
       # "1" means line tables only, which is useful for panic tracebacks.
       RUSTFLAGS: "-C debuginfo=1"
       # https://github.com/rust-lang/cargo/issues/10280
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
     steps:
       - checkout
       - rust_components
       - cache_restore
       - run:
           name: Install cargo-deny
           command: cargo install --force cargo-deny
       - run:
           name: cargo-deny Checks
           command: cargo deny check -s
       - cache_save
   check:
     docker:
       - image: quay.io/influxdb/rust:ci
     environment:
       # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
       CARGO_INCREMENTAL: "0"
       # Disable full debug symbol generation to speed up CI build
       # "1" means line tables only, which is useful for panic tracebacks.
       RUSTFLAGS: "-C debuginfo=1"
       # https://github.com/rust-lang/cargo/issues/10280
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
     steps:
       - checkout
       - rust_components
       - cache_restore
       - run:
           name: Install cargo-hack
           command: cargo install cargo-hack
       - run:
           name: Check all features
           command: cargo hack check --feature-powerset --no-dev-deps --workspace
       - cache_save
   doc:
     docker:
       - image: quay.io/influxdb/rust:ci
     environment:
       # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
       CARGO_INCREMENTAL: "0"
       # Disable full debug symbol generation to speed up CI build
       # "1" means line tables only, which is useful for panic tracebacks.
       RUSTFLAGS: "-C debuginfo=1"
       # https://github.com/rust-lang/cargo/issues/10280
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
     steps:
       - checkout
       - rust_components
       - cache_restore
       - run:
           name: Cargo doc
           # excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
           command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
       - cache_save
       - run:
           name: Compress Docs
           command: tar -cvzf rustdoc.tar.gz target/doc/
       - store_artifacts:
           path: rustdoc.tar.gz
   test:
     # setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker)
     docker:
       - image: quay.io/influxdb/rust:ci
       - image: localstack/localstack:0.14.4
       - image: mcr.microsoft.com/azure-storage/azurite
       - image: fsouza/fake-gcs-server
         command:
           - "-scheme"
           - "http"
     resource_class: 2xlarge # use of a smaller executor tends crashes on link
     environment:
       # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
       CARGO_INCREMENTAL: "0"
       # Disable full debug symbol generation to speed up CI build
       # "1" means line tables only, which is useful for panic tracebacks.
       RUSTFLAGS: "-C debuginfo=1"
       # https://github.com/rust-lang/cargo/issues/10280
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
       RUST_BACKTRACE: "1"
       # Run integration tests
       TEST_INTEGRATION: 1
       AWS_DEFAULT_REGION: "us-east-1"
       AWS_ACCESS_KEY_ID: test
       AWS_SECRET_ACCESS_KEY: test
       AWS_ENDPOINT: http://127.0.0.1:4566
       AZURE_USE_EMULATOR: "1"
       GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
       OBJECT_STORE_BUCKET: test-bucket
     steps:
       - run:
           name: Setup localstack (AWS emulation)
           command: |
             cd /tmp
             curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
             unzip awscliv2.zip
             sudo ./aws/install
             aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
       - run:
           name: Setup Azurite (Azure emulation)
           # the magical connection string is from https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
           command: |
             curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
             az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;'
       - run:
           name: Setup fake GCS server
           command: |
             curl -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
             echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
       - checkout
       - rust_components
       - cache_restore
       - run:
           name: Cargo test
           command: cargo test --workspace --features=aws,azure,azure_test,gcp
       - cache_save

   test_windows:
     executor:
       name: win/default
       size: medium
     environment:
       # https://github.com/rust-lang/cargo/issues/10280
       CARGO_NET_GIT_FETCH_WITH_CLI: "true"
     steps:
       - checkout
       - run:
           name: Download rustup
           command: wget https://win.rustup.rs/x86_64 -O rustup-init.exe
       - run:
           name: Install rustup
           command: .\rustup-init.exe -y --default-host=x86_64-pc-windows-msvc
       - run:
           name: Cargo test
           command: cargo test --workspace

 workflows:
   version: 2

   # CI for all pull requests.
   ci:
     jobs:
       - check
       - fmt
       - lint
       - cargo_audit
       - test
       - test_windows
       - doc
	---
	# CI Overview
	# -----------
	#
	# Each night:
	#
	# A build image is created (ci_image) from `docker/Dockerfile.ci` and is
	# pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
	# the CI tasks for the day.
	#
	# Every commit:
	#
	# The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
	#
	# On main if all these checks pass it will then additionally compile in "release" mode and
	# publish a docker image to quay.io/influxdb/iox:$COMMIT_SHA
	#
	# Manual CI Image:
	#
	# It is possible to manually trigger a rebuild of the image used in CI. To do this, navigate to
	# https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=main (overriding the
	# branch name if desired). Then:
	# - Click "Run Pipeline" in the top-right
	# - Expand "Add Parameters"
	# - Add a "boolean" parameter called "ci_image" with the value true
	# - Click "Run Pipeline"
	#
	# If you refresh the page you should see a newly running ci_image workflow
	#

	version: 2.1

	orbs:
	win: circleci/windows@4.1

	commands:
	rust_components:
	description: Verify installed components
	steps:
	- run:
	name: Verify installed components
	command: \|
	rustup --version
	rustup show
	cargo fmt --version
	cargo clippy --version

	cache_restore:
	description: Restore Cargo Cache
	steps:
	- restore_cache:
	name: Restoring Cargo Cache
	keys:
	- cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
	- cargo-cache-{{ arch }}-{{ .Branch }}
	- cargo-cache
	cache_save:
	description: Save Cargo Cache
	steps:
	- save_cache:
	name: Save Cargo Cache
	paths:
	- /usr/local/cargo/registry
	key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}

	jobs:
	fmt:
	docker:
	- image: quay.io/influxdb/rust:ci
	environment:
	# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
	CARGO_INCREMENTAL: "0"
	# Disable full debug symbol generation to speed up CI build
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	# https://github.com/rust-lang/cargo/issues/10280
	CARGO_NET_GIT_FETCH_WITH_CLI: "true"
	steps:
	- checkout
	- rust_components
	- cache_restore
	- run:
	name: Rust fmt
	command: cargo fmt --all -- --check
	- cache_save
	lint:
	docker:
	- image: quay.io/influxdb/rust:ci
	environment:
	# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
	CARGO_INCREMENTAL: "0"
	# Disable full debug symbol generation to speed up CI build
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	# https://github.com/rust-lang/cargo/issues/10280
	CARGO_NET_GIT_FETCH_WITH_CLI: "true"
	steps:
	- checkout
	- rust_components
	- cache_restore
	- run:
	name: Clippy
	command: cargo clippy --all-targets --all-features --workspace -- -D warnings
	- cache_save
	cargo_audit:
	docker:
	- image: quay.io/influxdb/rust:ci
	environment:
	# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
	CARGO_INCREMENTAL: "0"
	# Disable full debug symbol generation to speed up CI build
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	# https://github.com/rust-lang/cargo/issues/10280
	CARGO_NET_GIT_FETCH_WITH_CLI: "true"
	steps:
	- checkout
	- rust_components
	- cache_restore
	- run:
	name: Install cargo-deny
	command: cargo install --force cargo-deny
	- run:
	name: cargo-deny Checks
	command: cargo deny check -s
	- cache_save
	check:
	docker:
	- image: quay.io/influxdb/rust:ci
	environment:
	# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
	CARGO_INCREMENTAL: "0"
	# Disable full debug symbol generation to speed up CI build
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	# https://github.com/rust-lang/cargo/issues/10280
	CARGO_NET_GIT_FETCH_WITH_CLI: "true"
	steps:
	- checkout
	- rust_components
	- cache_restore
	- run:
	name: Install cargo-hack
	command: cargo install cargo-hack
	- run:
	name: Check all features
	command: cargo hack check --feature-powerset --no-dev-deps --workspace
	- cache_save
	doc:
	docker:
	- image: quay.io/influxdb/rust:ci
	environment:
	# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
	CARGO_INCREMENTAL: "0"
	# Disable full debug symbol generation to speed up CI build
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	# https://github.com/rust-lang/cargo/issues/10280
	CARGO_NET_GIT_FETCH_WITH_CLI: "true"
	steps:
	- checkout
	- rust_components
	- cache_restore
	- run:
	name: Cargo doc
	# excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
	command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
	- cache_save
	- run:
	name: Compress Docs
	command: tar -cvzf rustdoc.tar.gz target/doc/
	- store_artifacts:
	path: rustdoc.tar.gz
	test:
	# setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker)
	docker:
	- image: quay.io/influxdb/rust:ci
	- image: localstack/localstack:0.14.4
	- image: mcr.microsoft.com/azure-storage/azurite
	- image: fsouza/fake-gcs-server
	command:
	- "-scheme"
	- "http"
	resource_class: 2xlarge # use of a smaller executor tends crashes on link
	environment:
	# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
	CARGO_INCREMENTAL: "0"
	# Disable full debug symbol generation to speed up CI build
	# "1" means line tables only, which is useful for panic tracebacks.
	RUSTFLAGS: "-C debuginfo=1"
	# https://github.com/rust-lang/cargo/issues/10280
	CARGO_NET_GIT_FETCH_WITH_CLI: "true"
	RUST_BACKTRACE: "1"
	# Run integration tests
	TEST_INTEGRATION: 1
	AWS_DEFAULT_REGION: "us-east-1"
	AWS_ACCESS_KEY_ID: test
	AWS_SECRET_ACCESS_KEY: test
	AWS_ENDPOINT: http://127.0.0.1:4566
	AZURE_USE_EMULATOR: "1"
	GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"
	OBJECT_STORE_BUCKET: test-bucket
	steps:
	- run:
	name: Setup localstack (AWS emulation)
	command: \|
	cd /tmp
	curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
	unzip awscliv2.zip
	sudo ./aws/install
	aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
	- run:
	name: Setup Azurite (Azure emulation)
	# the magical connection string is from https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
	command: \|
	curl -sL https://aka.ms/InstallAzureCLIDeb \| sudo bash
	az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;'
	- run:
	name: Setup fake GCS server
	command: \|
	curl -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
	echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": ""}' > "$GOOGLE_SERVICE_ACCOUNT"
	- checkout
	- rust_components
	- cache_restore
	- run:
	name: Cargo test
	command: cargo test --workspace --features=aws,azure,azure_test,gcp
	- cache_save

	test_windows:
	executor:
	name: win/default
	size: medium
	environment:
	# https://github.com/rust-lang/cargo/issues/10280
	CARGO_NET_GIT_FETCH_WITH_CLI: "true"
	steps:
	- checkout
	- run:
	name: Download rustup
	command: wget https://win.rustup.rs/x86_64 -O rustup-init.exe
	- run:
	name: Install rustup
	command: .\rustup-init.exe -y --default-host=x86_64-pc-windows-msvc
	- run:
	name: Cargo test
	command: cargo test --workspace

	workflows:
	version: 2

	# CI for all pull requests.
	ci:
	jobs:
	- check
	- fmt
	- lint
	- cargo_audit
	- test
	- test_windows
	- doc