blob: f37b2d3740a4b2f0692020f122dff41d3cb8b2ca [file] [log] [blame]
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This file is broken into multiple sections delimited by ---. Each section specifies a set of
# reference encodings for a single standardized coder used in a specific context.
#
# Each section contains up to 3 properties:
#
# coder: a common coder spec. Currently, a URN and URNs for component coders as necessary.
# nested: a boolean meaning whether the coder was used in the nested context. Missing means to
# test both contexts, a shorthand for when the coder is invariant across context.
# examples: a map of {encoded bytes: original JSON object} encoded with the coder in the context.
# The LHS (key) is a byte array encoded as a JSON-escaped string. The RHS (value) is
# one of a few standard JSON types such as numbers, strings, dicts that map naturally
# to the type encoded by the coder.
#
# These choices were made to strike a balance between portability, ease of use, and simple
# legibility of this file itself.
#
# It is expected that future work will move the `coder` field into a format that it would be
# represented by the Runner API, so that it can be understood by all SDKs and harnesses.
#
# If a coder is marked non-deterministic in the coder spec, then only the decoding should be validated.
coder:
urn: "urn:beam:coders:bytes:0.1"
nested: false
examples:
"abc": abc
"ab\0c": "ab\0c"
---
coder:
urn: "urn:beam:coders:bytes:0.1"
nested: true
examples:
"\u0003abc": abc
"\u0004ab\0c": "ab\0c"
"\u00c8\u0001 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|":
" 10| 20| 30| 40| 50| 60| 70| 80| 90| 100| 110| 120| 130| 140| 150| 160| 170| 180| 190| 200|"
---
coder:
urn: "urn:beam:coders:varint:0.1"
examples:
"\0": 0
"\u0001": 1
"\u000A": 10
"\u00c8\u0001": 200
"\u00e8\u0007": 1000
"\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u0001": -1
---
coder:
urn: "urn:beam:coders:kv:0.1"
components: [{urn: "urn:beam:coders:bytes:0.1"},
{urn: "urn:beam:coders:varint:0.1"}]
examples:
"\u0003abc\0": {key: abc, value: 0}
"\u0004ab\0c\u000A": {key: "ab\0c", value: 10}
---
coder:
urn: "urn:beam:coders:kv:0.1"
components: [{urn: "urn:beam:coders:bytes:0.1"},
{urn: "urn:beam:coders:bytes:0.1"}]
nested: false
examples:
"\u0003abcdef": {key: abc, value: def}
"\u0004ab\0cde\0f": {key: "ab\0c", value: "de\0f"}
---
coder:
urn: "urn:beam:coders:kv:0.1"
components: [{urn: "urn:beam:coders:bytes:0.1"},
{urn: "urn:beam:coders:bytes:0.1"}]
nested: true
examples:
"\u0003abc\u0003def": {key: abc, value: def}
"\u0004ab\0c\u0004de\0f": {key: "ab\0c", value: "de\0f"}
---
coder:
urn: "urn:beam:coders:interval_window:0.1"
examples:
"\u0080\u0000\u0001\u0052\u009a\u00a4\u009b\u0068\u0080\u00dd\u00db\u0001" : {end: 1454293425000, span: 3600000}
"\u0080\u0000\u0001\u0053\u0034\u00ec\u0074\u00e8\u0080\u0090\u00fb\u00d3\u0009" : {end: 1456881825000, span: 2592000000}
"\u007f\u00df\u003b\u0064\u005a\u001c\u00ad\u0076\u00ed\u0002" : {end: -9223372036854410, span: 365}
"\u0080\u0020\u00c4\u009b\u00a5\u00e3\u0053\u00f7\u0000" : {end: 9223372036854775, span: 0}
---
coder:
urn: "urn:beam:coders:stream:0.1"
components: [{urn: "urn:beam:coders:varint:0.1"}]
examples:
"\0\0\0\u0001\0": [0]
"\0\0\0\u0004\u0001\n\u00c8\u0001\u00e8\u0007": [1, 10, 200, 1000]
"\0\0\0\0": []
---
coder:
urn: "urn:beam:coders:stream:0.1"
components: [{urn: "urn:beam:coders:bytes:0.1"}]
examples:
"\0\0\0\u0001\u0003abc": ["abc"]
"\0\0\0\u0002\u0004ab\0c\u0004de\0f": ["ab\0c", "de\0f"]
"\0\0\0\0": []
---
coder:
urn: "urn:beam:coders:stream:0.1"
components: [{urn: "urn:beam:coders:bytes:0.1"}]
# This is for iterables of unknown length, where the encoding is not
# deterministic.
non_deterministic: True
examples:
"\u00ff\u00ff\u00ff\u00ff\u0000": []
"\u00ff\u00ff\u00ff\u00ff\u0001\u0003abc\u0000": ["abc"]
"\u00ff\u00ff\u00ff\u00ff\u0002\u0004ab\u0000c\u0004de\u0000f\u0000": ["ab\0c", "de\0f"]
---
coder:
urn: "urn:beam:coders:stream:0.1"
components: [{urn: "urn:beam:coders:global_window:0.1"}]
examples:
"\0\0\0\u0001": [""]
---
coder:
urn: "urn:beam:coders:global_window:0.1"
examples:
"": ""
---
# All windowed values consist of pane infos that represent NO_FIRING until full support is added
# in the Python SDK (BEAM-1522).
coder:
urn: "urn:beam:coders:windowed_value:0.1"
components: [{urn: "urn:beam:coders:varint:0.1"},
{urn: "urn:beam:coders:global_window:0.1"}]
examples:
"\u0080\0\u0001R\u009a\u00a4\u009bh\0\0\0\u0001\u000f\u0002": {
value: 2,
timestamp: 1454293425000,
pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
windows: ["global"]
}
---
coder:
urn: "urn:beam:coders:windowed_value:0.1"
components: [{urn: "urn:beam:coders:varint:0.1"},
{urn: "urn:beam:coders:interval_window:0.1"}]
examples:
"\u007f\u00ff\u00ff\u00ff\u00ff\u00f9\u00e5\u0080\0\0\0\u0001\u0080\0\u0001R\u009a\u00a4\u009bh\u00c0\u008b\u0011\u000f\u0004": {
value: 4,
timestamp: -400000,
pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
windows: [{end: 1454293425000, span: 280000}]
}
"\u007f\u00ff\u00ff\u00ff\u00ff\u00ff\u00ff\u009c\0\0\0\u0002\u0080\0\u0001R\u009a\u00a4\u009bh\u0080\u00dd\u00db\u0001\u007f\u00df;dZ\u001c\u00adv\u00ed\u0002\u000f\u0002": {
value: 2,
timestamp: -100,
pane: {is_first: True, is_last: True, timing: UNKNOWN, index: 0, on_time_index: 0},
windows: [{end: 1454293425000, span: 3600000}, {end: -9223372036854410, span: 365}]
}