#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

{% from 'shared_jinja.yml' import pipeline_prefix with context %}
{% from 'shared_jinja.yml' import github_access with context %}
{% from 'shared_jinja.yml' import init_retry with context %}

---

{% macro plan_resource_gets(test) %}
- get: geode-ci
  passed:
  - Build
- aggregate:
  - get: geode
    trigger: true
    passed:
    - Build
  - get: geode-build-version
    passed:
    - Build
{% endmacro %}

{%- macro deep_merge(a, b): %}
  {%- if b is defined %}
    {%- for k,v in b.items(): %}
      {%- if v is not defined: %}
        {%- do a.pop(k) %}
      {%- else: %}
        {%- if v is mapping: %}
          {%- if a[k] is not mapping: %}
            {%- do a.update({ k: { } }) %}
          {%- endif %}
          {%- do deep_merge(a[k], v) %}
        {%- else: %}
          {%- do a.update({ k: v }) %}
        {%- endif %}
      {% endif %}
    {%- endfor %}
  {%- endif %}
{%- endmacro %}

{%- do deep_merge(a, b) %}

{% macro common_instance_params(test) -%}
GCP_PROJECT: ((gcp-project))
CPUS: {{ test.CPUS }}
RAM: {{ test.RAM }}
{%- endmacro %}

{% macro common_test_params(test) -%}
  {%- if test.CALL_STACK_TIMEOUT -%}
CALL_STACK_TIMEOUT: {{ test.CALL_STACK_TIMEOUT }}
  {%- endif %}
  {%- if test.GRADLE_TASK_OPTIONS -%}
GRADLE_TASK_OPTIONS: {{ test.GRADLE_TASK_OPTIONS }}
  {%- endif %}
DUNIT_PARALLEL_FORKS: {{ test.DUNIT_PARALLEL_FORKS }}
MAINTENANCE_VERSION: ((geode-build-branch ))
BUILD_BRANCH: ((geode-build-branch))
PARALLEL_DUNIT: {{ test.PARALLEL_DUNIT }}
{% if test.PARALLEL_GRADLE is defined %}
PARALLEL_GRADLE: {{ test.PARALLEL_GRADLE }}
{% else %}
PARALLEL_GRADLE: true
{% endif %}
ARTIFACT_BUCKET: ((artifact-bucket))
SERVICE_ACCOUNT: ((concourse-gcp-account))
GRADLE_GLOBAL_ARGS: ((gradle-global-args))
{%- endmacro %}

{% macro all_gating_jobs() %}
  {%- for test in (tests) if not test.name=="StressNew" -%}
  {%- for java_test_version in (java_test_versions) %}
- {{test.name}}Test{{java_test_version.name}}
  {%- endfor -%}
  {%- endfor -%}
{% endmacro %}

groups:
- name: main
  jobs:
  - {{ build_test.name }}
  {{- all_gating_jobs() | indent(2) }}
  - UpdatePassingTokens
  {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
  - PublishArtifacts
  {%- endif %}
  {%- for flavor in (benchmarks.flavors) %}
  - Benchmark{{flavor.title}}
  {%- endfor %}
- name: complete
  jobs:
  - {{ build_test.name }}
  {%- for test in (tests) if not test.name=="StressNew" -%}
    {%- for java_test_version in (java_test_versions) %}
  - {{test.name}}Test{{java_test_version.name}}
    {%- endfor -%}
  {%- endfor %}
  - UpdatePassingTokens
  {%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
  - PublishArtifacts
  {%- endif %}
  {%- for flavor in (benchmarks.flavors) %}
  - Benchmark{{flavor.title}}
  {%- endfor %}
- name: linux
  jobs:
  - {{ build_test.name }}
  {%- for test in (tests) if test.PLATFORM=="linux" and not test.name=="StressNew" -%}
    {% for java_test_version in (java_test_versions) %}
  - {{test.name}}Test{{java_test_version.name}}
    {%- endfor -%}
  {%- endfor %}
  {%- for flavor in (benchmarks.flavors) %}
  - Benchmark{{flavor.title}}
  {%- endfor %}
- name: windows
  jobs:
  - {{ build_test.name }}
  {%- for test in (tests) if test.PLATFORM=="windows" -%}
    {% for java_test_version in (java_test_versions) %}
  - {{test.name}}Test{{java_test_version.name}}
    {%- endfor -%}
  {%- endfor %}
{%- for java_test_version in (java_test_versions) %}
- name: {{java_test_version.name}}
  jobs:
  - {{ build_test.name }}
  {%- for test in (tests) if not test.name=="StressNew" %}
  - {{test.name}}Test{{java_test_version.name}}
  {%- endfor -%}
{%- endfor %}
- name: Semver Management
  jobs:
  {%- for semverPiece in ['major', 'minor', 'patch'] %}
  - Bump{{ semverPiece.title() }}
  {%- endfor %}

resources:
- name: concourse-metadata-resource
  type: concourse-metadata-resource
  source: {}
- name: geode
  type: git
  source:
    branch: ((geode-build-branch))
    ignore_paths:
    - ci/*
    - dev-tools/release/*
    - "*.md"
    {{ github_access() | indent(4) }}
- name: geode-ci
  type: git
  source:
    branch: ((geode-build-branch))
    depth: 1
    paths:
    - ci/pipelines/geode-build/*
    - ci/scripts/*
    {{ github_access() | indent(4) }}
- name: geode-benchmarks
  type: git
  source:
    branch: {{benchmarks.benchmark_branch}}
    depth: 1
    uri: https://github.com/apache/geode-benchmarks.git
- name: geode-build-version
  type: semver
  source:
    bucket: ((artifact-bucket))
    driver: gcs
    initial_version: {{ metadata.initial_version }}
    json_key: ((!concourse-gcp-key))
    key: semvers/((pipeline-prefix))((geode-build-branch))/version
- name: geode-passing-tokens
  type: gcs-resource
  source:
    bucket: ((artifact-bucket))
    json_key: ((concourse-gcp-key))
    versioned_file: semvers/((pipeline-prefix))((geode-build-branch))/passing-build-tokens.json
- name: alpine-tools-image
  type: docker-image
  source:
    username: ((docker-username))
    password: ((docker-password))
    repository: gcr.io/((gcp-project))/((pipeline-prefix))alpine-tools
    tag: latest

resource_types:
- name: concourse-metadata-resource
  type: docker-image
  source:
    password: ((docker-password))
    repository: gcr.io/((gcp-project))/((pipeline-prefix))concourse-metadata-resource
    tag: latest
    username: ((docker-username))
- name: gcs-resource
  type: docker-image
  source:
    repository: frodenas/gcs-resource
jobs:
{% for semverPiece in ['major', 'minor', 'patch'] -%}
- name: Bump{{ semverPiece.title() }}
  serial: true
  plan:
  - get: geode-build-version
    params: { bump: {{ semverPiece }} }
  - put: geode-build-version
    params:
      file: geode-build-version/number
{% endfor %}
- name: {{build_test.name}}
  public: true
  serial: true
  plan:
  - aggregate:
    - get: geode-ci
    - get: alpine-tools-image
    - get: geode
      trigger: true
    - get: geode-build-version
      params:
        pre: ((semver-prerelease-token))
  - do:
    - put: concourse-metadata-resource
    {{ init_retry()|indent(4) }}
    - task: create_instance
      image: alpine-tools-image
      config:
        platform: linux
        params:
          {{ common_instance_params(build_test) | indent(10) }}
          GEODE_BRANCH: {{repository.branch}}
          GEODE_FORK: {{repository.fork}}
          JAVA_BUILD_VERSION: {{ java_build_version.version }}
        run:
          path: geode-ci/ci/scripts/create_instance.sh
        inputs:
        - name: attempts-log
          path: old
        - name: concourse-metadata-resource
        - name: geode
        - name: geode-ci
        outputs:
        - name: attempts-log
          path: new
        - name: instance-data
      timeout: 15m
      attempts: 10
  - aggregate:
    - put: geode-build-version
      params:
        file: geode-build-version/number
    - task: rsync_code_up
      image: alpine-tools-image
      config:
        platform: linux
        run:
          path: geode-ci/ci/scripts/rsync_code_up.sh
        inputs:
        - name: geode-ci
        - name: geode
        - name: instance-data
    timeout: 10m
    on_failure:
      do:
      - task: delete_instance
        image: alpine-tools-image
        config:
          platform: linux
          run:
            path: geode-ci/ci/scripts/delete_instance.sh
          inputs:
          - name: geode-ci
          - name: instance-data
        timeout: 1h
  - task: build
    image: alpine-tools-image
    config:
      platform: linux
      params:
        JAVA_BUILD_VERSION: {{ java_build_version.version }}
        GRADLE_TASK: {{ build_test.GRADLE_TASK }}
        {{ common_test_params(build_test) | indent(8) }}
      run:
        path: geode-ci/ci/scripts/execute_build.sh
      inputs:
      - name: geode-ci
      - name: geode
      - name: instance-data
      - name: geode-build-version
      outputs:
      - name: built-geode
      - name: results
    timeout: {{build_test.EXECUTE_TEST_TIMEOUT}}
    ensure:
      do:
      - task: rsync_code_down
        image: alpine-tools-image
        config:
          platform: linux
          params:
            JAVA_BUILD_VERSION: {{ java_build_version.version }}
          run:
            path: geode-ci/ci/scripts/rsync_code_down.sh
          inputs:
          - name: geode-ci
          - name: instance-data
          outputs:
          - name: geode-results
        timeout: 5m
      ensure:
        do:
        - aggregate:
          - task: archive_results
            image: alpine-tools-image
            config:
              platform: linux
              params:
                ARTIFACT_SLUG: {{build_test.ARTIFACT_SLUG}}-{{java_build_version.name}}
                GRADLE_TASK: {{build_test.GRADLE_TASK}}
                MAINTENANCE_VERSION: ((geode-build-branch))
                ARTIFACT_BUCKET: ((artifact-bucket))
                SERVICE_ACCOUNT: ((!concourse-gcp-account))
                TAR_GEODE_BUILD_ARTIFACTS: true
              run:
                path: geode-ci/ci/scripts/archive_results.sh
              inputs:
              - name: concourse-metadata-resource
              - name: geode-ci
              - name: geode-build-version
              - name: geode-results
            timeout: 1h
          - task: delete_instance
            image: alpine-tools-image
            config:
              platform: linux
              run:
                path: geode-ci/ci/scripts/delete_instance.sh
              inputs:
              - name: geode-ci
              - name: instance-data
            timeout: 1h

- name: UpdatePassingTokens
  public: true
  serial: true
  plan:
  - aggregate:
    - get: alpine-tools-image
    - get: geode
      passed: &update-token-passed-anchor
{%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
      - PublishArtifacts
{% else %}
      {% for flavor in (benchmarks.flavors) %}
      - Benchmark{{flavor.title}}
      {% endfor %}
{% endif %}
      trigger: true
    - get: geode-build-version
      trigger: true
      passed: *update-token-passed-anchor
  - task: couple-sha-and-build-id
    image: alpine-tools-image
    config:
      platform: linux
      inputs:
      - name: geode
      - name: geode-build-version
      outputs:
      - name: geode-passing-tokens
      run:
        path: bash
        args:
        - -cx
        - |
          pushd geode
            GEODE_SHA=$(git rev-parse HEAD)
          popd
          GEODE_SEMVER=$(cat geode-build-version/number)

          GS_PATH=gs://((artifact-bucket))/semvers/((pipeline-prefix))((geode-build-branch))/passing-build-tokens.json
          CURRENT_PASSING_SHA=$(gsutil cat ${GS_PATH} | jq -r .ref)
          set -e
          # Check that the incoming GEODE_SHA is a descendent of the currently stored value.
          # Keeps us from winding back the repository in the case of an out-of-order pipeline pass
          if [ -z "${CURRENT_PASSING_SHA}" ] || (cd geode; git merge-base --is-ancestor ${CURRENT_PASSING_SHA} ${GEODE_SHA}); then
            cat > geode-passing-tokens/passing-build-tokens.json <<JSON
          {
            "ref": "${GEODE_SHA}",
            "semver": "${GEODE_SEMVER}"
          }
          JSON
          fi
  - aggregate:
    - put: geode-passing-tokens
      params:
        file: geode-passing-tokens/passing-build-tokens.json

{% for run_var in (benchmarks.flavors) %}
- name: Benchmark{{ run_var.title }}
  public: true
  max_in_flight: 3
  plan:
  - get: geode-ci
    passed:
    {{ all_gating_jobs() | indent(4) }}
  - get: alpine-tools-image
  - aggregate:
    - get: geode
      passed:
      {{ all_gating_jobs() | indent(6) }}
      trigger: true
    - get: geode-benchmarks
    - get: geode-build-version
      passed:
      {{ all_gating_jobs() | indent(6) }}
    - put: concourse-metadata-resource
  - do:
    - task: run_benchmarks{{ run_var.title }}
      image: alpine-tools-image
      config:
        platform: linux
        params:
          AWS_ACCESS_KEY_ID: ((benchmarks-access-key-id))
          AWS_SECRET_ACCESS_KEY: ((benchmarks-secret-access-key))
          AWS_DEFAULT_REGION: us-west-2
          AWS_REGION: us-west-2
          ARTIFACT_BUCKET: ((artifact-bucket))
          BENCHMARKS_BRANCH: {{benchmarks.benchmark_branch}}
          BASELINE_BRANCH: {{benchmarks.baseline_branch}}
          BASELINE_VERSION: {{benchmarks.baseline_version}}
          FLAGS: {{ run_var.flag }}
          TAG_POSTFIX: {{ run_var.title }}
          TEST_OPTIONS: {{ run_var.options }}
        run:
          path: geode-ci/ci/scripts/run_benchmarks.sh
        inputs:
        - name: geode
        - name: geode-ci
        - name: geode-benchmarks
        - name: concourse-metadata-resource
        outputs:
        - name: results
      timeout: 8h
      ensure:
        do:
        - task: cleanup_benchmarks
          image: alpine-tools-image
          config:
            platform: linux
            params:
              AWS_ACCESS_KEY_ID: ((benchmarks-access-key-id))
              AWS_SECRET_ACCESS_KEY: ((benchmarks-secret-access-key))
              AWS_DEFAULT_REGION: us-west-2
              AWS_REGION: us-west-2
              ARTIFACT_BUCKET: ((artifact-bucket))
              BASELINE_BRANCH: {{benchmarks.baseline_branch}}
              BASELINE_VERSION: {{benchmarks.baseline_version}}
              FLAGS: {{ run_var.flag }}
              TAG_POSTFIX: {{ run_var.title }}
              TEST_OPTIONS: {{ run_var.options }}
            run:
              path: geode-ci/ci/scripts/cleanup_benchmarks.sh
            inputs:
            - name: geode
            - name: geode-ci
            - name: geode-benchmarks
            - name: concourse-metadata-resource
            - name: results
{% endfor %}
{% if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- name: PublishArtifacts
  public: true
  plan:
  - aggregate:
    - get: geode-ci
      passed: &publish-passed-inputs
      {% for flavor in (benchmarks.flavors) %}
      - Benchmark{{flavor.title}}
      {% endfor %}
    - get: alpine-tools-image
    - get: geode
      passed: *publish-passed-inputs
      trigger: true
  - aggregate:
    - get: geode-build-version
      trigger: true
      passed: *publish-passed-inputs
    - put: concourse-metadata-resource
    {{ init_retry()|indent(4) }}
  - task: create_instance
    image: alpine-tools-image
    config:
      platform: linux
      params:
        {{ common_instance_params(publish_artifacts) | indent(8) }}
        GEODE_BRANCH: {{repository.branch}}
        GEODE_FORK: {{repository.fork}}
        JAVA_BUILD_VERSION: {{ java_build_version.version }}
      run:
        path: geode-ci/ci/scripts/create_instance.sh
      inputs:
      - name: concourse-metadata-resource
      - name: geode-ci
      - name: geode
      - name: attempts-log
        path: old
      outputs:
      - name: instance-data
      - name: attempts-log
        path: new
    timeout: 15m
    attempts: 10
  - task: rsync_code_up
    image: alpine-tools-image
    config:
      platform: linux
      run:
        path: geode-ci/ci/scripts/rsync_code_up.sh
      inputs:
      - name: geode-ci
      - name: geode
      - name: instance-data
    timeout: 5m
  - task: publish
    image: alpine-tools-image
    config:
      platform: linux
      params:
        MAINTENANCE_VERSION: ((geode-build-branch))
        ARTIFACT_BUCKET: ((artifact-bucket))
        SERVICE_ACCOUNT: ((!concourse-gcp-account))
        JAVA_BUILD_VERSION: {{ java_build_version.version }}
        MAVEN_SNAPSHOT_BUCKET: ((maven-snapshot-bucket))
        GRADLE_GLOBAL_ARGS: ((gradle-global-args))
      run:
        path: geode-ci/ci/scripts/execute_publish.sh
      inputs:
      - name: instance-data
      - name: geode
      - name: geode-ci
      - name: geode-build-version
    ensure:
      do:
      - task: delete_instance
        image: alpine-tools-image
        config:
          platform: linux
          run:
            path: geode-ci/ci/scripts/delete_instance.sh
          inputs:
          - name: geode-ci
          - name: instance-data
        timeout: 1h

{% endif %}

{%- for test in tests if not test.name=="StressNew" %}
  {%- set parameters = {} %}
  {%- do deep_merge(parameters, test) %}
  {%- for java_test_version in (java_test_versions) %}
    {%- if java_test_version.override is defined and java_test_version.override[test.name] is defined %}
      {%- do deep_merge(parameters, java_test_version.override[test.name]) %}
    {%- endif %}
- name: {{test.name}}Test{{java_test_version.name}}
  public: true
  plan:
  - do:
    {{- plan_resource_gets(test) |indent(4) }}
      - put: concourse-metadata-resource
      - get: alpine-tools-image
    - aggregate:
      - do:
        {{ init_retry()|indent(8) }}
        - task: create_instance-{{java_test_version.name}}
          image: alpine-tools-image
          config:
            platform: linux
            params:
              {{ common_instance_params(parameters) | indent(14) }}
              GEODE_BRANCH: {{repository.branch}}
              GEODE_FORK: {{repository.fork}}
              JAVA_BUILD_VERSION: {{ java_build_version.version }}
              JAVA_TEST_VERSION: {{ java_test_version.version }}
            run:
              path: geode-ci/ci/scripts/create_instance.sh
            inputs:
            - name: concourse-metadata-resource
            - name: geode-ci
            - name: geode
            - name: attempts-log
              path: old
            outputs:
            - name: instance-data-{{java_test_version.name}}
              path: instance-data
            - name: attempts-log
              path: new
          timeout: 15m
          attempts: 10
        - task: rsync_code_up-{{java_test_version.name}}
          image: alpine-tools-image
          config:
            platform: linux
            run:
              path: geode-ci/ci/scripts/rsync_code_up.sh
            inputs:
            - name: geode-ci
            - name: geode
            - name: instance-data-{{java_test_version.name}}
              path: instance-data
          timeout: 15m
          attempts: 10
        - task: execute_tests-{{java_test_version.name}}
          image: alpine-tools-image
          config:
            platform: linux
            params:
              ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}-{{java_test_version.name}}
              JAVA_BUILD_VERSION: {{ java_build_version.version }}
              JAVA_TEST_VERSION: {{ java_test_version.version }}
              GRADLE_TASK: {{test.GRADLE_TASK}}
              {{ common_test_params(parameters) | indent(14) }}
            run:
              path: geode-ci/ci/scripts/execute_tests.sh
            inputs:
            - name: geode-ci
            - name: geode
            - name: instance-data-{{java_test_version.name}}
              path: instance-data
          timeout: {{parameters.EXECUTE_TEST_TIMEOUT}}
        ensure:
          do:
          - task: rsync_code_down-{{java_test_version.name}}
            image: alpine-tools-image
            config:
              platform: linux
              params:
                JAVA_BUILD_VERSION: {{ java_build_version.version }}
                ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}-{{java_test_version.name}}
              run:
                path: geode-ci/ci/scripts/rsync_code_down.sh
              inputs:
              - name: geode-ci
              - name: instance-data-{{java_test_version.name}}
                path: instance-data
              outputs:
              - name: geode-results-{{java_test_version.name}}
                path: geode-results
            timeout: 15m
            attempts: 10
          ensure:
            do:
            - aggregate:
              - task: archive_results-{{java_test_version.name}}
                image: alpine-tools-image
                config:
                  platform: linux
                  params:
                    ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}-{{java_test_version.name}}
                    GRADLE_TASK: {{test.GRADLE_TASK}}
                    MAINTENANCE_VERSION: ((geode-build-branch))
                    ARTIFACT_BUCKET: ((artifact-bucket))
                    SERVICE_ACCOUNT: ((!concourse-gcp-account))
                  run:
                    path: geode-ci/ci/scripts/archive_results.sh
                  inputs:
                  - name: concourse-metadata-resource
                  - name: geode-ci
                  - name: geode-build-version
                  - name: geode-results-{{java_test_version.name}}
                    path: geode-results
                timeout: 1h
              - task: delete_instance-{{java_test_version.name}}
                image: alpine-tools-image
                config:
                  platform: linux
                  run:
                    path: geode-ci/ci/scripts/delete_instance.sh
                  inputs:
                  - name: geode-ci
                  - name: instance-data-{{java_test_version.name}}
                    path: instance-data
                timeout: 1h
{% endfor -%}
{% endfor -%}