sdks/python/apache_beam/yaml/pipeline.schema.yaml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

$schema: 'http://json-schema.org/draft-07/schema#'
$id: https://github.com/apache/beam/tree/master/sdks/python/apache_beam/yaml/pipeline.schema.yaml

$defs:

  transformBase:
    type: object
    properties:
      type: { type: string }
      name: { type: string }
      windowing:
        $ref: '#/$defs/windowing'
      resource_hints:
        $ref: '#/$defs/resource_hints'
      input:
        oneOf:
          - type: string
          - type: array
            items: { type: string }
          - type: object
            properties: { __line__: {}}
            additionalProperties:
              type: string
      output:
        oneOf:
          - type: string
          - type: object
            properties: { __line__: {}}
            additionalProperties:
              type: string
      output_schema:
        type: object
    additionalProperties: true
    required:
      - type

  nestedTransform:
    allOf:
      - type: object
        properties:
          type: {}
          name: {}
          input: {}
          output: {}
          windowing: {}
          resource_hints: {}
          transforms:
            type: array
            items:
              $ref: '#/$defs/transform'
          extra_transforms:
            type: array
            items:
              $ref: '#/$defs/transform'
          source:
            $ref: '#/$defs/transform'
          sink:
            $ref: '#/$defs/transform'
          __line__: {}
          __uuid__: {}
        additionalProperties: false
      - anyOf:
        - required:
          - transforms
        - required:
          - source
          - sink

  implicitInputOutputs:
    properties:
      transforms:
        type: array
        items:
          properties:
            # Must be implicit.
            input: { not: {} }
            output: { not: {} }
      source:
        properties:
          # Must be implicit.
          input: { not: {} }
          output: { not: {} }
      sink:
        properties:
          # Must be implicit.
          input: { not: {} }
          output: { not: {} }

  transform:
    allOf:
      - $ref: '#/$defs/transformBase'
      - if:
          properties: { type: { const: composite }}
        then:
          allOf:
            - $ref: '#/$defs/nestedTransform'
            - properties:
                extra_transforms: { not: {} }
      - if:
          properties: { type: { const: chain }}
        then:
          allOf:
            - $ref: '#/$defs/nestedTransform'
            - $ref: '#/$defs/implicitInputOutputs'

      - if:
          not:
            anyOf:
              - properties: { type: { const: composite }}
              - properties: { type: { const: chain }}
        then:
          properties:
            type: {}
            name: {}
            input: {}
            output: {}
            output_schema: { type: object }
            windowing: {}
            resource_hints: {}
            config: { type: object }
            __line__: {}
            __uuid__: {}
          additionalProperties: false

  windowing: {}  # TODO

  resource_hints:
    type: object

  provider:
    # TODO(robertwb): Consider enumerating the provider types along with
    # the arguments they accept/expect (possibly in a separate schema file).
    type: object
    properties:
      type: { type: string }
      transforms:
        type: object
        properties: { __line__: {}}
        additionalProperties:
          type: string
      config: { type: object }
      __line__: {}
      __uuid__: {}
    additionalProperties: false
    required:
      - type
      - transforms
      - config

  providerInclude:
    # TODO(robertwb): Consider enumerating the provider types along with
    # the arguments they accept/expect (possibly in a separate schema file).
    type: object
    properties:
      include: { type: string }
      __line__: {}
      __uuid__: {}
    additionalProperties: false
    required:
      - include

  providerOrProviderInclude:
    if:
      allOf: [
        { properties: { include: { type: string }}},
        { required: [ "include" ] }
      ]
    then:
      $ref: '#/$defs/providerInclude'
    else:
      $ref: '#/$defs/provider'

  testElementsSpec:
    type: object
    properties:
      name: { type: string }
      elements:
        type: array
        item:
          type: object
      __line__: {}
      __uuid__: {}
    additionalProperties: false

  testSpec:
    type: object
    properties:
      name: { type: string }
      mock_inputs:
        type: array
        items:
          $ref: '#/$defs/testElementsSpec'
      mock_outputs:
        type: array
        items:
          $ref: '#/$defs/testElementsSpec'
      expected_inputs:
        type: array
        items:
          $ref: '#/$defs/testElementsSpec'
      expected_outputs:
        type: array
        items:
          $ref: '#/$defs/testElementsSpec'
      allowed_sources:
        type: array
        items:
          { type: string }
      __line__: {}
      __uuid__: {}
    additionalProperties: false


type: object
properties:
  pipeline:
    allOf:
      # These are the only top-level properties defined in pipeline.
      - type: object
        properties:
          type:
            oneOf:
              - { const: composite }
              - { const: chain }
          windowing:
            $ref: '#/$defs/windowing'
          resource_hints:
            $ref: '#/$defs/resource_hints'
          transforms: {}
          extra_transforms: {}
          sink: {}
          source: {}
          __line__: {}
          __uuid__: {}
        additionalProperties: false
      # This defines the allowable contents of the attributes above.
      - $ref: '#/$defs/nestedTransform'
      # A chain-type transform, like a chain composite, must have implicit io.
      - if:
          properties: { type: { const: chain }}
          required: [type]
        then:
          $ref: '#/$defs/implicitInputOutputs'
  providers:
    type: array
    items:
      $ref: '#/$defs/providerOrProviderInclude'
  options:
    type: object
  tests:
    type: array
    items:
      $ref: '#/$defs/testSpec'
required:
  - pipeline