blob: a400a4a31787aed24ac1417ead004fc7f05e7fda [file] [log] [blame]
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
{% from 'shared_jinja.yml' import pipeline_prefix with context %}
{% from 'shared_jinja.yml' import github_access with context %}
{% from 'shared_jinja.yml' import init_retry with context %}
---
{% macro plan_resource_gets(test) %}
- get: geode-ci
passed:
- Build
- aggregate:
- get: geode
trigger: true
passed:
- Build
- get: geode-build-version
passed:
- Build
{% endmacro %}
{%- macro deep_merge(a, b): %}
{%- if b is defined %}
{%- for k,v in b.items(): %}
{%- if v is not defined: %}
{%- do a.pop(k) %}
{%- else: %}
{%- if v is mapping: %}
{%- if a[k] is not mapping: %}
{%- do a.update({ k: { } }) %}
{%- endif %}
{%- do deep_merge(a[k], v) %}
{%- else: %}
{%- do a.update({ k: v }) %}
{%- endif %}
{% endif %}
{%- endfor %}
{%- endif %}
{%- endmacro %}
{%- do deep_merge(a, b) %}
{% macro common_instance_params(test) -%}
GCP_PROJECT: ((gcp-project))
CPUS: {{ test.CPUS }}
RAM: {{ test.RAM }}
{%- endmacro %}
{% macro common_test_params(test) -%}
{%- if test.CALL_STACK_TIMEOUT -%}
CALL_STACK_TIMEOUT: {{ test.CALL_STACK_TIMEOUT }}
{%- endif %}
{%- if test.GRADLE_TASK_OPTIONS -%}
GRADLE_TASK_OPTIONS: {{ test.GRADLE_TASK_OPTIONS }}
{%- endif %}
DUNIT_PARALLEL_FORKS: {{ test.DUNIT_PARALLEL_FORKS }}
MAINTENANCE_VERSION: ((geode-build-branch ))
BUILD_BRANCH: ((geode-build-branch))
PARALLEL_DUNIT: {{ test.PARALLEL_DUNIT }}
{% if test.PARALLEL_GRADLE is defined %}
PARALLEL_GRADLE: {{ test.PARALLEL_GRADLE }}
{% else %}
PARALLEL_GRADLE: true
{% endif %}
ARTIFACT_BUCKET: ((artifact-bucket))
SERVICE_ACCOUNT: ((concourse-gcp-account))
GRADLE_GLOBAL_ARGS: ((gradle-global-args))
{%- endmacro %}
{% macro all_gating_jobs() %}
{%- for test in (tests) if not test.name=="StressNew" -%}
{%- for java_test_version in (java_test_versions) %}
- {{test.name}}Test{{java_test_version.name}}
{%- endfor -%}
{%- endfor -%}
{% endmacro %}
groups:
- name: main
jobs:
- {{ build_test.name }}
{{- all_gating_jobs() | indent(2) }}
- UpdatePassingTokens
{%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- PublishArtifacts
{%- endif %}
- Benchmark
- name: complete
jobs:
- {{ build_test.name }}
{%- for test in (tests) if not test.name=="StressNew" -%}
{%- for java_test_version in (java_test_versions) %}
- {{test.name}}Test{{java_test_version.name}}
{%- endfor -%}
{%- endfor %}
- UpdatePassingTokens
{%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- PublishArtifacts
{%- endif %}
- Benchmark
- name: linux
jobs:
- {{ build_test.name }}
{%- for test in (tests) if test.PLATFORM=="linux" and not test.name=="StressNew" -%}
{% for java_test_version in (java_test_versions) %}
- {{test.name}}Test{{java_test_version.name}}
{%- endfor -%}
{%- endfor %}
- Benchmark
- name: windows
jobs:
- {{ build_test.name }}
{%- for test in (tests) if test.PLATFORM=="windows" -%}
{% for java_test_version in (java_test_versions) %}
- {{test.name}}Test{{java_test_version.name}}
{%- endfor -%}
{%- endfor %}
{%- for java_test_version in (java_test_versions) %}
- name: {{java_test_version.name}}
jobs:
- {{ build_test.name }}
{%- for test in (tests) if not test.name=="StressNew" %}
- {{test.name}}Test{{java_test_version.name}}
{%- endfor -%}
{%- endfor %}
- name: Semver Management
jobs:
{%- for semverPiece in ['major', 'minor', 'patch'] %}
- Bump{{ semverPiece.title() }}
{%- endfor %}
resources:
- name: concourse-metadata-resource
type: concourse-metadata-resource
source: {}
- name: geode
type: git
source:
branch: ((geode-build-branch))
ignore_paths:
- ci/*
- dev-tools/release/*
- "*.md"
{{ github_access() | indent(4) }}
- name: geode-ci
type: git
source:
branch: ((geode-build-branch))
depth: 1
paths:
- ci/pipelines/geode-build/*
- ci/scripts/*
{{ github_access() | indent(4) }}
- name: geode-benchmarks
type: git
source:
branch: {{benchmarks.benchmark_branch}}
depth: 1
uri: https://github.com/apache/geode-benchmarks.git
- name: geode-build-version
type: semver
source:
bucket: ((artifact-bucket))
driver: gcs
initial_version: {{ metadata.initial_version }}
json_key: ((!concourse-gcp-key))
key: semvers/((pipeline-prefix))((geode-build-branch))/version
- name: geode-passing-tokens
type: gcs-resource
source:
bucket: ((artifact-bucket))
json_key: ((concourse-gcp-key))
versioned_file: semvers/((pipeline-prefix))((geode-build-branch))/passing-build-tokens.json
- name: alpine-tools-image
type: docker-image
source:
username: ((docker-username))
password: ((docker-password))
repository: gcr.io/((gcp-project))/((pipeline-prefix))alpine-tools
tag: latest
resource_types:
- name: concourse-metadata-resource
type: docker-image
source:
password: ((docker-password))
repository: gcr.io/((gcp-project))/((pipeline-prefix))concourse-metadata-resource
tag: latest
username: ((docker-username))
- name: gcs-resource
type: docker-image
source:
repository: frodenas/gcs-resource
jobs:
{% for semverPiece in ['major', 'minor', 'patch'] -%}
- name: Bump{{ semverPiece.title() }}
serial: true
plan:
- get: geode-build-version
params: { bump: {{ semverPiece }} }
- put: geode-build-version
params:
file: geode-build-version/number
{% endfor %}
- name: {{build_test.name}}
public: true
serial: true
plan:
- aggregate:
- get: geode-ci
- get: alpine-tools-image
- get: geode
trigger: true
- get: geode-build-version
params:
pre: ((semver-prerelease-token))
- do:
- put: concourse-metadata-resource
{{ init_retry()|indent(4) }}
- task: create_instance
image: alpine-tools-image
config:
platform: linux
params:
{{ common_instance_params(build_test) | indent(10) }}
GEODE_BRANCH: {{repository.branch}}
GEODE_FORK: {{repository.fork}}
JAVA_BUILD_VERSION: {{ java_build_version.version }}
run:
path: geode-ci/ci/scripts/create_instance.sh
inputs:
- name: attempts-log
path: old
- name: concourse-metadata-resource
- name: geode
- name: geode-ci
outputs:
- name: attempts-log
path: new
- name: instance-data
timeout: 15m
attempts: 10
- aggregate:
- put: geode-build-version
params:
file: geode-build-version/number
- task: rsync_code_up
image: alpine-tools-image
config:
platform: linux
run:
path: geode-ci/ci/scripts/rsync_code_up.sh
inputs:
- name: geode-ci
- name: geode
- name: instance-data
timeout: 10m
on_failure:
do:
- task: delete_instance
image: alpine-tools-image
config:
platform: linux
run:
path: geode-ci/ci/scripts/delete_instance.sh
inputs:
- name: geode-ci
- name: instance-data
timeout: 1h
- task: build
image: alpine-tools-image
config:
platform: linux
params:
JAVA_BUILD_VERSION: {{ java_build_version.version }}
GRADLE_TASK: {{ build_test.GRADLE_TASK }}
{{ common_test_params(build_test) | indent(8) }}
run:
path: geode-ci/ci/scripts/execute_build.sh
inputs:
- name: geode-ci
- name: geode
- name: instance-data
- name: geode-build-version
outputs:
- name: built-geode
- name: results
timeout: {{build_test.EXECUTE_TEST_TIMEOUT}}
ensure:
do:
- task: rsync_code_down
image: alpine-tools-image
config:
platform: linux
params:
JAVA_BUILD_VERSION: {{ java_build_version.version }}
run:
path: geode-ci/ci/scripts/rsync_code_down.sh
inputs:
- name: geode-ci
- name: instance-data
outputs:
- name: geode-results
timeout: 5m
ensure:
do:
- aggregate:
- task: archive_results
image: alpine-tools-image
config:
platform: linux
params:
ARTIFACT_SLUG: {{build_test.ARTIFACT_SLUG}}-{{java_build_version.name}}
GRADLE_TASK: {{build_test.GRADLE_TASK}}
MAINTENANCE_VERSION: ((geode-build-branch))
ARTIFACT_BUCKET: ((artifact-bucket))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
TAR_GEODE_BUILD_ARTIFACTS: true
run:
path: geode-ci/ci/scripts/archive_results.sh
inputs:
- name: concourse-metadata-resource
- name: geode-ci
- name: geode-build-version
- name: geode-results
timeout: 1h
- task: delete_instance
image: alpine-tools-image
config:
platform: linux
run:
path: geode-ci/ci/scripts/delete_instance.sh
inputs:
- name: geode-ci
- name: instance-data
timeout: 1h
- name: UpdatePassingTokens
public: true
serial: true
plan:
- aggregate:
- get: alpine-tools-image
- get: geode
passed: &update-token-passed-anchor
{%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- PublishArtifacts
{% else %}
- Benchmark
{% endif %}
trigger: true
- get: geode-build-version
trigger: true
passed: *update-token-passed-anchor
- task: couple-sha-and-build-id
image: alpine-tools-image
config:
platform: linux
inputs:
- name: geode
- name: geode-build-version
outputs:
- name: geode-passing-tokens
run:
path: bash
args:
- -cx
- |-
pushd geode
GEODE_SHA=$(git rev-parse HEAD)
popd
GEODE_SEMVER=$(cat geode-build-version/number)
GS_PATH=gs://((artifact-bucket))/semvers/((pipeline-prefix))((geode-build-branch))/passing-build-tokens.json
CURRENT_PASSING_SHA=$(gsutil cat ${GS_PATH} | jq -r .ref)
set -e
# Check that the incoming GEODE_SHA is a descendent of the currently stored value.
# Keeps us from winding back the repository in the case of an out-of-order pipeline pass
if [ -z "${CURRENT_PASSING_SHA}" ] || (cd geode; git merge-base --is-ancestor ${CURRENT_PASSING_SHA} ${GEODE_SHA}); then
cat > geode-passing-tokens/passing-build-tokens.json <<JSON
{
"ref": "${GEODE_SHA}",
"semver": "${GEODE_SEMVER}"
}
JSON
fi
- aggregate:
- put: geode-passing-tokens
params:
file: geode-passing-tokens/passing-build-tokens.json
- name: Benchmark
public: true
max_in_flight: 3
plan:
- get: geode-ci
passed:
{{ all_gating_jobs() | indent(4) }}
- aggregate:
- get: alpine-tools-image
- get: geode
passed:
{{ all_gating_jobs() | indent(6) }}
trigger: true
- get: geode-benchmarks
- get: geode-build-version
passed:
{{ all_gating_jobs() | indent(6) }}
- put: concourse-metadata-resource
- aggregate:
{% for run_var in [{'title': '', 'flag':'', 'options': ''}, {'title': '_with_ssl', 'flag': '-PwithSsl', 'options': '--tests=*GetBenchmark --tests=*PutBenchmark'}, {'title': '_with_security_manager', 'flag': '-PwithSecurityManager', 'options': '--tests=Partitioned*'}] %}
- task: run_benchmarks{{ run_var.title }}
image: alpine-tools-image
config:
platform: linux
params:
AWS_ACCESS_KEY_ID: ((benchmarks-access-key-id))
AWS_SECRET_ACCESS_KEY: ((benchmarks-secret-access-key))
AWS_DEFAULT_REGION: us-west-2
AWS_REGION: us-west-2
ARTIFACT_BUCKET: ((artifact-bucket))
BENCHMARKS_BRANCH: {{benchmarks.benchmark_branch}}
BASELINE_BRANCH: {{benchmarks.baseline_branch}}
BASELINE_VERSION: {{benchmarks.baseline_version}}
FLAGS: {{ run_var.flag }}
TAG_POSTFIX: {{ run_var.title }}
TEST_OPTIONS: {{ run_var.options }}
run:
path: geode-ci/ci/scripts/run_benchmarks.sh
inputs:
- name: geode
- name: geode-ci
- name: geode-benchmarks
- name: concourse-metadata-resource
outputs:
- name: results
timeout: 8h
ensure:
do:
- task: cleanup_benchmarks
image: alpine-tools-image
config:
platform: linux
params:
AWS_ACCESS_KEY_ID: ((benchmarks-access-key-id))
AWS_SECRET_ACCESS_KEY: ((benchmarks-secret-access-key))
AWS_DEFAULT_REGION: us-west-2
AWS_REGION: us-west-2
ARTIFACT_BUCKET: ((artifact-bucket))
BASELINE_BRANCH: {{benchmarks.baseline_branch}}
BASELINE_VERSION: {{benchmarks.baseline_version}}
FLAGS: {{ run_var.flag }}
TAG_POSTFIX: {{ run_var.title }}
TEST_OPTIONS: {{ run_var.options }}
run:
path: geode-ci/ci/scripts/cleanup_benchmarks.sh
inputs:
- name: geode
- name: geode-ci
- name: geode-benchmarks
- name: concourse-metadata-resource
- name: results
{% endfor %}
{% if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- name: PublishArtifacts
public: true
plan:
- aggregate:
- get: geode-ci
passed:
- Benchmark
- get: alpine-tools-image
- get: geode
passed:
- Benchmark
trigger: true
- aggregate:
- get: geode-build-version
passed:
- Benchmark
- do:
- put: concourse-metadata-resource
{{ init_retry()|indent(6) }}
- task: create_instance
image: alpine-tools-image
config:
platform: linux
params:
{{ common_instance_params(publish_artifacts) | indent(12) }}
GEODE_BRANCH: {{repository.branch}}
GEODE_FORK: {{repository.fork}}
JAVA_BUILD_VERSION: {{ java_build_version.version }}
run:
path: geode-ci/ci/scripts/create_instance.sh
inputs:
- name: concourse-metadata-resource
- name: geode-ci
- name: geode
- name: attempts-log
path: old
outputs:
- name: instance-data
- name: attempts-log
path: new
timeout: 15m
attempts: 10
- task: rsync_code_up
image: alpine-tools-image
config:
platform: linux
run:
path: geode-ci/ci/scripts/rsync_code_up.sh
inputs:
- name: geode-ci
- name: geode
- name: instance-data
timeout: 5m
- task: publish
image: alpine-tools-image
config:
platform: linux
params:
MAINTENANCE_VERSION: ((geode-build-branch))
ARTIFACT_BUCKET: ((artifact-bucket))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
JAVA_BUILD_VERSION: {{ java_build_version.version }}
MAVEN_SNAPSHOT_BUCKET: ((maven-snapshot-bucket))
GRADLE_GLOBAL_ARGS: ((gradle-global-args))
run:
path: geode-ci/ci/scripts/execute_publish.sh
inputs:
- name: instance-data
- name: geode
- name: geode-ci
- name: geode-build-version
ensure:
do:
- task: delete_instance
image: alpine-tools-image
config:
platform: linux
run:
path: geode-ci/ci/scripts/delete_instance.sh
inputs:
- name: geode-ci
- name: instance-data
timeout: 1h
{% endif %}
{%- for test in tests if not test.name=="StressNew" %}
{%- set parameters = {} %}
{%- do deep_merge(parameters, test) %}
{%- for java_test_version in (java_test_versions) %}
{%- if java_test_version.override is defined and java_test_version.override[test.name] is defined %}
{%- do deep_merge(parameters, java_test_version.override[test.name]) %}
{%- endif %}
- name: {{test.name}}Test{{java_test_version.name}}
public: true
plan:
- do:
{{- plan_resource_gets(test) |indent(4) }}
- put: concourse-metadata-resource
- get: alpine-tools-image
- aggregate:
- do:
{{ init_retry()|indent(8) }}
- task: create_instance-{{java_test_version.name}}
image: alpine-tools-image
config:
platform: linux
params:
{{ common_instance_params(parameters) | indent(14) }}
GEODE_BRANCH: {{repository.branch}}
GEODE_FORK: {{repository.fork}}
JAVA_BUILD_VERSION: {{ java_build_version.version }}
JAVA_TEST_VERSION: {{ java_test_version.version }}
run:
path: geode-ci/ci/scripts/create_instance.sh
inputs:
- name: concourse-metadata-resource
- name: geode-ci
- name: geode
- name: attempts-log
path: old
outputs:
- name: instance-data-{{java_test_version.name}}
path: instance-data
- name: attempts-log
path: new
timeout: 15m
attempts: 10
- task: rsync_code_up-{{java_test_version.name}}
image: alpine-tools-image
config:
platform: linux
run:
path: geode-ci/ci/scripts/rsync_code_up.sh
inputs:
- name: geode-ci
- name: geode
- name: instance-data-{{java_test_version.name}}
path: instance-data
timeout: 15m
attempts: 10
- task: execute_tests-{{java_test_version.name}}
image: alpine-tools-image
config:
platform: linux
params:
ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}-{{java_test_version.name}}
JAVA_BUILD_VERSION: {{ java_build_version.version }}
JAVA_TEST_VERSION: {{ java_test_version.version }}
GRADLE_TASK: {{test.GRADLE_TASK}}
{{ common_test_params(parameters) | indent(14) }}
run:
path: geode-ci/ci/scripts/execute_tests.sh
inputs:
- name: geode-ci
- name: geode
- name: instance-data-{{java_test_version.name}}
path: instance-data
timeout: {{parameters.EXECUTE_TEST_TIMEOUT}}
ensure:
do:
- task: rsync_code_down-{{java_test_version.name}}
image: alpine-tools-image
config:
platform: linux
params:
JAVA_BUILD_VERSION: {{ java_build_version.version }}
ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}-{{java_test_version.name}}
run:
path: geode-ci/ci/scripts/rsync_code_down.sh
inputs:
- name: geode-ci
- name: instance-data-{{java_test_version.name}}
path: instance-data
outputs:
- name: geode-results-{{java_test_version.name}}
path: geode-results
timeout: 15m
attempts: 10
ensure:
do:
- aggregate:
- task: archive_results-{{java_test_version.name}}
image: alpine-tools-image
config:
platform: linux
params:
ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}-{{java_test_version.name}}
GRADLE_TASK: {{test.GRADLE_TASK}}
MAINTENANCE_VERSION: ((geode-build-branch))
ARTIFACT_BUCKET: ((artifact-bucket))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
run:
path: geode-ci/ci/scripts/archive_results.sh
inputs:
- name: concourse-metadata-resource
- name: geode-ci
- name: geode-build-version
- name: geode-results-{{java_test_version.name}}
path: geode-results
timeout: 1h
- task: delete_instance-{{java_test_version.name}}
image: alpine-tools-image
config:
platform: linux
run:
path: geode-ci/ci/scripts/delete_instance.sh
inputs:
- name: geode-ci
- name: instance-data-{{java_test_version.name}}
path: instance-data
timeout: 1h
{% endfor -%}
{% endfor -%}