| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| /* |
| * Protocol Buffers describing the Fn API and boostrapping. |
| * |
| * TODO: Usage of plural names in lists looks awkward in Java |
| * e.g. getOutputsMap, addCodersBuilder |
| * |
| * TODO: gRPC / proto field names conflict with generated code |
| * e.g. "class" in java, "output" in python |
| */ |
| |
| syntax = "proto3"; |
| |
| /* TODO: Consider consolidating common components in another package |
| * and language namespaces for re-use with Runner Api. |
| */ |
| |
| package org.apache.beam.model.fn_execution.v1; |
| |
| option go_package = "fnexecution_v1"; |
| option java_package = "org.apache.beam.model.fnexecution.v1"; |
| option java_outer_classname = "BeamFnApi"; |
| |
| import "beam_runner_api.proto"; |
| import "endpoints.proto"; |
| import "google/protobuf/descriptor.proto"; |
| import "google/protobuf/timestamp.proto"; |
| import "google/protobuf/wrappers.proto"; |
| import "metrics.proto"; |
| |
| /* |
| * Constructs that define the pipeline shape. |
| * |
| * These are mostly unstable due to the missing pieces to be shared with |
| * the Runner Api like windowing strategy, display data, .... There are still |
| * some modelling questions related to whether a side input is modelled |
| * as another field on a PrimitiveTransform or as part of inputs and we |
| * still are missing things like the CompositeTransform. |
| */ |
| |
| // A representation of an input or output definition on a primitive transform. |
| // Stable |
| message Target { |
| // A repeated list of target definitions. |
| message List { |
| repeated Target target = 1; |
| } |
| |
| // (Required) The id of the PrimitiveTransform which is the target. |
| string primitive_transform_reference = 1; |
| |
| // (Required) The local name of an input or output defined on the primitive |
| // transform. |
| string name = 2; |
| } |
| |
| // A descriptor for connecting to a remote port using the Beam Fn Data API. |
| // Allows for communication between two environments (for example between the |
| // runner and the SDK). |
| // Stable |
| message RemoteGrpcPort { |
| // (Required) An API descriptor which describes where to |
| // connect to including any authentication that is required. |
| org.apache.beam.model.pipeline.v1.ApiServiceDescriptor |
| api_service_descriptor = 1; |
| |
| // (Required) The ID of the Coder that will be used to encode and decode data |
| // sent over this port. |
| string coder_id = 2; |
| } |
| |
| /* |
| * Control Plane API |
| * |
| * Progress reporting and splitting still need further vetting. Also, this may |
| * change with the addition of new types of instructions/responses related to |
| * metrics. |
| */ |
| |
| // An API that describes the work that a SDK harness is meant to do. |
| // Stable |
| service BeamFnControl { |
| // Instructions sent by the runner to the SDK requesting different types |
| // of work. |
| rpc Control( |
| // A stream of responses to instructions the SDK was asked to be |
| // performed. |
| stream InstructionResponse) |
| returns ( |
| // A stream of instructions requested of the SDK to be performed. |
| stream InstructionRequest) {} |
| } |
| |
| // A request sent by a runner which the SDK is asked to fulfill. |
| // For any unsupported request type, an error should be returned with a |
| // matching instruction id. |
| // Stable |
| message InstructionRequest { |
| // (Required) An unique identifier provided by the runner which represents |
| // this requests execution. The InstructionResponse MUST have the matching id. |
| string instruction_id = 1; |
| |
| // (Required) A request that the SDK Harness needs to interpret. |
| oneof request { |
| RegisterRequest register = 1000; |
| ProcessBundleRequest process_bundle = 1001; |
| ProcessBundleProgressRequest process_bundle_progress = 1002; |
| ProcessBundleSplitRequest process_bundle_split = 1003; |
| FinalizeBundleRequest finalize_bundle = 1004; |
| } |
| } |
| |
| // The response for an associated request the SDK had been asked to fulfill. |
| // Stable |
| message InstructionResponse { |
| // (Required) A reference provided by the runner which represents a requests |
| // execution. The InstructionResponse MUST have the matching id when |
| // responding to the runner. |
| string instruction_id = 1; |
| |
| // If this is specified, then this instruction has failed. |
| // A human readable string representing the reason as to why processing has |
| // failed. |
| string error = 2; |
| |
| // If the instruction did not fail, it is required to return an equivalent |
| // response type depending on the request this matches. |
| oneof response { |
| RegisterResponse register = 1000; |
| ProcessBundleResponse process_bundle = 1001; |
| ProcessBundleProgressResponse process_bundle_progress = 1002; |
| ProcessBundleSplitResponse process_bundle_split = 1003; |
| FinalizeBundleResponse finalize_bundle = 1004; |
| } |
| } |
| |
| // A list of objects which can be referred to by the runner in |
| // future requests. |
| // Stable |
| message RegisterRequest { |
| // (Optional) The set of descriptors used to process bundles. |
| repeated ProcessBundleDescriptor process_bundle_descriptor = 1; |
| } |
| |
| // Stable |
| message RegisterResponse {} |
| |
| // Definitions that should be used to construct the bundle processing graph. |
| message ProcessBundleDescriptor { |
| // (Required) A pipeline level unique id which can be used as a reference to |
| // refer to this. |
| string id = 1; |
| |
| // (Required) A map from pipeline-scoped id to PTransform. |
| map<string, org.apache.beam.model.pipeline.v1.PTransform> transforms = 2; |
| |
| // (Required) A map from pipeline-scoped id to PCollection. |
| map<string, org.apache.beam.model.pipeline.v1.PCollection> pcollections = 3; |
| |
| // (Required) A map from pipeline-scoped id to WindowingStrategy. |
| map<string, org.apache.beam.model.pipeline.v1.WindowingStrategy> |
| windowing_strategies = 4; |
| |
| // (Required) A map from pipeline-scoped id to Coder. |
| map<string, org.apache.beam.model.pipeline.v1.Coder> coders = 5; |
| |
| // (Required) A map from pipeline-scoped id to Environment. |
| map<string, org.apache.beam.model.pipeline.v1.Environment> environments = 6; |
| |
| // A descriptor describing the end point to use for State API |
| // calls. Required if the Runner intends to send remote references over the |
| // data plane or if any of the transforms rely on user state or side inputs. |
| org.apache.beam.model.pipeline.v1.ApiServiceDescriptor |
| state_api_service_descriptor = 7; |
| } |
| |
| // One of the applications specifying the scope of work for a bundle. |
| // See |
| // https://docs.google.com/document/d/1tUDb45sStdR8u7-jBkGdw3OGFK7aa2-V7eo86zYSE_4/edit#heading=h.9g3g5weg2u9 |
| // for further details. |
| message BundleApplication { |
| // (Required) The primitive transform to which to pass the element |
| string ptransform_id = 1; |
| |
| // (Required) Name of the transform's input to which to pass the element. |
| string input_id = 2; |
| |
| // (Required) The encoded element to pass to the transform. |
| bytes element = 3; |
| |
| // The map is keyed by the local output name of the PTransform. Each |
| // value represents a lower bound on the timestamps of elements that |
| // are produced by this PTransform into each of its output PCollections |
| // when invoked with this application. |
| map<string, google.protobuf.Timestamp> output_watermarks = 4; |
| |
| // Represents an estimate for the amount of currently outstanding work. |
| message Backlog { |
| // This informs Runners on how to aggregate the backlog |
| // being reported across multiple active bundles. Backlogs |
| // are aggregated using the set of partitions. |
| // |
| // For example SplittableDoFn's which consume elements from: |
| // * a globally shared resource such as a Pubsub queue should set this |
| // to “”. |
| // * a shared partitioned resource should use the partition identifier. |
| // * a uniquely partitioned resource such as a file range should set this |
| // to |
| // file name + start offset. |
| bytes partition = 1; |
| |
| // The estimate for the backlog. |
| oneof value { |
| // Represents an estimate for the amount of outstanding work. Values |
| // compare lexicographically. |
| bytes bytes = 1000; |
| |
| // Whether the backlog is unknown. |
| bool is_unknown = 1001; |
| } |
| } |
| |
| // (Required) An estimate for the amount outstanding work related to |
| // this application. |
| Backlog backlog = 5; |
| |
| // (Required) Whether this application potentially produces an unbounded |
| // amount of data. Note that this should only be set to BOUNDED if and |
| // only if the application is known to produce a finite amount of output. |
| // |
| // Note that this is different from the backlog as the backlog represents |
| // how much work there is currently outstanding. |
| org.apache.beam.model.pipeline.v1.IsBounded.Enum is_bounded = 6; |
| |
| // Contains additional monitoring information related to this application. |
| // |
| // Each application is able to report information that some runners |
| // will use consume when providing a UI or for making scaling and performance |
| // decisions. See https://s.apache.org/beam-bundles-backlog-splitting for |
| // details about what types of signals may be useful to report. |
| repeated org.apache.beam.model.pipeline.v1.MonitoringInfo monitoring_infos = 7; |
| } |
| |
| // An Application should be scheduled for execution after a delay. |
| message DelayedBundleApplication { |
| // Recommended time at which the application should be scheduled to execute |
| // by the runner. Times in the past may be scheduled to execute immediately. |
| google.protobuf.Timestamp requested_execution_time = 1; |
| |
| // (Required) The application that should be scheduled. |
| BundleApplication application = 2; |
| } |
| |
| // A request to process a given bundle. |
| // Stable |
| message ProcessBundleRequest { |
| // (Required) A reference to the process bundle descriptor that must be |
| // instantiated and executed by the SDK harness. |
| string process_bundle_descriptor_reference = 1; |
| |
| // (Optional) A list of cache tokens that can be used by an SDK to reuse |
| // cached data returned by the State API across multiple bundles. |
| repeated bytes cache_tokens = 2; |
| } |
| |
| message ProcessBundleResponse { |
| // (Optional) If metrics reporting is supported by the SDK, this represents |
| // the final metrics to record for this bundle. |
| // DEPRECATED |
| Metrics metrics = 1; |
| |
| // (Optional) Specifies that the bundle has not been completed and the |
| // following applications need to be scheduled and executed in the future. |
| // A runner that does not yet support residual roots MUST still check that |
| // this is empty for correctness. |
| repeated DelayedBundleApplication residual_roots = 2; |
| |
| // (Required) The list of metrics or other MonitoredState |
| // collected while processing this bundle. |
| repeated org.apache.beam.model.pipeline.v1.MonitoringInfo monitoring_infos = 3; |
| |
| // (Optional) Specifies that the runner must callback to this worker |
| // once the output of the bundle is committed. The Runner must send a |
| // FinalizeBundleRequest with the instruction id of the ProcessBundleRequest |
| // that is related to this ProcessBundleResponse. |
| bool requires_finalization = 4; |
| } |
| |
| // A request to report progress information for a given bundle. |
| // This is an optional request to be handled and is used to support advanced |
| // SDK features such as SplittableDoFn, user level metrics etc. |
| message ProcessBundleProgressRequest { |
| // (Required) A reference to an active process bundle request with the given |
| // instruction id. |
| string instruction_reference = 1; |
| } |
| |
| // DEPRECATED |
| message Metrics { |
| // PTransform level metrics. |
| // These metrics are split into processed and active element groups for |
| // progress reporting purposes. This allows a Runner to see what is measured, |
| // what is estimated and what can be extrapolated to be able to accurately |
| // estimate the backlog of remaining work. |
| message PTransform { |
| // Metrics that are measured for processed and active element groups. |
| message Measured { |
| // (Optional) Map from local input name to number of elements processed |
| // from this input. |
| // If unset, assumed to be the sum of the outputs of all producers to |
| // this transform (for ProcessedElements) and 0 (for ActiveElements). |
| map<string, int64> input_element_counts = 1; |
| |
| // (Required) Map from local output name to number of elements produced |
| // for this output. |
| map<string, int64> output_element_counts = 2; |
| |
| // (Optional) The total time spent so far in processing the elements in |
| // this group, in seconds. |
| double total_time_spent = 3; |
| |
| // TODO: Add other element group level metrics. |
| } |
| |
| // Metrics for fully processed elements. |
| message ProcessedElements { |
| // (Required) |
| Measured measured = 1; |
| } |
| |
| // Metrics for active elements. |
| // An element is considered active if the SDK has started but not finished |
| // processing it yet. |
| message ActiveElements { |
| // (Required) |
| Measured measured = 1; |
| |
| // Estimated metrics. |
| |
| // (Optional) Sum of estimated fraction of known work remaining for all |
| // active elements, as reported by this transform. |
| // If not reported, a Runner could extrapolate this from the processed |
| // elements. |
| // TODO: Handle the case when known work is infinite. |
| double fraction_remaining = 2; |
| |
| // (Optional) Map from local output name to sum of estimated number |
| // of elements remaining for this output from all active elements, |
| // as reported by this transform. |
| // If not reported, a Runner could extrapolate this from the processed |
| // elements. |
| map<string, int64> output_elements_remaining = 3; |
| } |
| |
| // (Required): Metrics for processed elements. |
| ProcessedElements processed_elements = 1; |
| // (Required): Metrics for active elements. |
| ActiveElements active_elements = 2; |
| |
| // (Optional): Map from local output name to its watermark. |
| // The watermarks reported are tentative, to get a better sense of progress |
| // while processing a bundle but before it is committed. At bundle commit |
| // time, a Runner needs to also take into account the timers set to compute |
| // the actual watermarks. |
| map<string, int64> watermarks = 3; |
| |
| repeated User user = 4; |
| // TODO: Define other transform level system metrics. |
| } |
| |
| // User defined metrics |
| message User { |
| // A key for identifying a metric at the most granular level. |
| message MetricName { |
| // (Required): The namespace of this metric. |
| string namespace = 2; |
| |
| // (Required): The name of this metric. |
| string name = 3; |
| } |
| |
| // Data associated with a counter metric. |
| message CounterData { |
| int64 value = 1; |
| } |
| |
| // Data associated with a distribution metric. |
| message DistributionData { |
| int64 count = 1; |
| int64 sum = 2; |
| int64 min = 3; |
| int64 max = 4; |
| } |
| |
| // Data associated with a Gauge metric. |
| message GaugeData { |
| int64 value = 1; |
| google.protobuf.Timestamp timestamp = 2; |
| } |
| |
| // (Required) The identifier for this metric. |
| MetricName metric_name = 1; |
| |
| // (Required) The data for this metric. |
| oneof data { |
| CounterData counter_data = 1001; |
| DistributionData distribution_data = 1002; |
| GaugeData gauge_data = 1003; |
| } |
| } |
| |
| map<string, PTransform> ptransforms = 1; |
| } |
| |
| message ProcessBundleProgressResponse { |
| // DEPRECATED (Required) |
| Metrics metrics = 1; |
| |
| // (Required) The list of metrics or other MonitoredState |
| // collected while processing this bundle. |
| repeated org.apache.beam.model.pipeline.v1.MonitoringInfo monitoring_infos = 3; |
| |
| // The list of currently active primary roots that are being |
| // executed. Required to be populated for PTransforms which can be split. |
| repeated BundleApplication primary_roots = 4; |
| } |
| |
| // Represents a request to the SDK to split a currently active bundle. |
| message ProcessBundleSplitRequest { |
| // (Required) A reference to an active process bundle request with the given |
| // instruction id. |
| string instruction_reference = 1; |
| |
| // (Required) Specifies that the Runner would like the bundle to split itself |
| // such that it performs no more work than the backlog specified for each |
| // PTransform. The interpretation of how much work should be processed is up |
| // to the PTransform. |
| // |
| // For example, A backlog of "" tells the SDK to perform as little work as |
| // possible, effectively checkpointing when able. The remaining backlog |
| // will be relative to the backlog reported during processing. |
| // |
| // If the backlog is unspecified for a PTransform, the runner would like |
| // the SDK to process all data received for that PTransform. |
| map<string, bytes> backlog_remaining = 2; |
| |
| // A message specifying the desired split for a single transform. |
| message DesiredSplit { |
| // (Required) The fraction of known work remaining in this bundle |
| // for this transform that should be kept by the SDK after this split. |
| // |
| // Set to 0 to "checkpoint" as soon as possible (keeping as little work as |
| // possible and returning the remainder). |
| float fraction_of_remainder = 1; |
| |
| // A set of allowed element indices where the SDK may split. When this is |
| // empty, there are no constraints on where to split. |
| repeated int64 allowed_split_points = 3; |
| |
| // (Required for GrpcRead operations) Number of total elements expected |
| // to be sent to this GrpcRead operation, required to correctly account |
| // for unreceived data when determining where to split. |
| int64 estimated_input_elements = 2; |
| |
| // TODO(SDF): Allow providing weights rather than sizes. |
| } |
| |
| // (Required) Specifies the desired split for each transform. |
| // |
| // Currently only splits at GRPC read operations are supported. |
| // This may, of course, limit the amount of work downstream operations |
| // receive. |
| map<string, DesiredSplit> desired_splits = 3; |
| } |
| |
| // Represents a partition of the bundle: a "primary" and |
| // a "residual", with the following properties: |
| // - The work in primary and residual doesn't overlap, and combined, adds up |
| // to the work in the current bundle if the split hadn't happened. |
| // - The current bundle, if it keeps executing, will have done none of the |
| // work under residual_roots. |
| // - The current bundle, if no further splits happen, will have done exactly |
| // the work under primary_roots. |
| // For more rigorous definitions see https://s.apache.org/beam-breaking-fusion |
| message ProcessBundleSplitResponse { |
| // Root applications that should replace the current bundle. |
| repeated BundleApplication primary_roots = 1; |
| |
| // Root applications that have been removed from the current bundle and |
| // have to be executed in a separate bundle (e.g. in parallel on a different |
| // worker, or after the current bundle completes, etc.) |
| repeated DelayedBundleApplication residual_roots = 2; |
| |
| // Represents contiguous portions of the data channel that are either |
| // entirely processed or entirely unprocessed and belong to the primary |
| // or residual respectively. |
| // |
| // This affords both a more efficient representation over the FnAPI |
| // (if the bundle is large) and often a more efficient representation |
| // on the runner side (e.g. if the set of elements can be represented |
| // as some range in an underlying dataset). |
| message ChannelSplit { |
| // (Required) The grpc read transform reading this channel. |
| string ptransform_id = 1; |
| |
| // (Required) Name of the transform's input to which to pass the element. |
| string input_id = 2; |
| |
| // The last element of the input channel that should be entirely considered |
| // part of the primary, identified by its absolute index in the (ordered) |
| // channel. |
| int32 last_primary_element = 3; |
| |
| // The first element of the input channel that should be entirely considered |
| // part of the residual, identified by its absolute index in the (ordered) |
| // channel. |
| int32 first_residual_element = 4; |
| } |
| |
| // Partitions of input data channels into primary and residual elements, |
| // if any. Should not include any elements represented in the bundle |
| // applications roots above. |
| repeated ChannelSplit channel_splits = 3; |
| } |
| |
| |
| message FinalizeBundleRequest { |
| // (Required) A reference to a completed process bundle request with the given |
| // instruction id. |
| string instruction_reference = 1; |
| } |
| |
| message FinalizeBundleResponse { |
| // Empty |
| } |
| |
| /* |
| * Data Plane API |
| */ |
| |
| // Messages used to represent logical byte streams. |
| // Stable |
| message Elements { |
| // Represents multiple encoded elements in nested context for a given named |
| // instruction and target. |
| message Data { |
| // (Required) A reference to an active instruction request with the given |
| // instruction id. |
| string instruction_reference = 1; |
| |
| // (Required) A definition representing a consumer or producer of this data. |
| // If received by a harness, this represents the consumer within that |
| // harness that should consume these bytes. If sent by a harness, this |
| // represents the producer of these bytes. |
| // |
| // Note that a single element may span multiple Data messages. |
| // |
| // Note that a sending/receiving pair should share the same target |
| // identifier. |
| Target target = 2; |
| |
| // (Optional) Represents a part of a logical byte stream. Elements within |
| // the logical byte stream are encoded in the nested context and |
| // concatenated together. |
| // |
| // An empty data block represents the end of stream for the given |
| // instruction and target. |
| bytes data = 3; |
| } |
| |
| // (Required) A list containing parts of logical byte streams. |
| repeated Data data = 1; |
| } |
| |
| // Stable |
| service BeamFnData { |
| // Used to send data between harnesses. |
| rpc Data( |
| // A stream of data representing input. |
| stream Elements) |
| returns ( |
| // A stream of data representing output. |
| stream Elements) {} |
| } |
| |
| /* |
| * State API |
| */ |
| |
| message StateRequest { |
| // (Required) An unique identifier provided by the SDK which represents this |
| // requests execution. The StateResponse corresponding with this request |
| // will have the matching id. |
| string id = 1; |
| |
| // (Required) The associated instruction id of the work that is currently |
| // being processed. This allows for the runner to associate any modifications |
| // to state to be committed with the appropriate work execution. |
| string instruction_reference = 2; |
| |
| // (Required) The state key this request is for. |
| StateKey state_key = 3; |
| |
| // (Required) The action to take on this request. |
| oneof request { |
| // A request to get state. |
| StateGetRequest get = 1000; |
| |
| // A request to append to state. |
| StateAppendRequest append = 1001; |
| |
| // A request to clear state. |
| StateClearRequest clear = 1002; |
| } |
| } |
| |
| message StateResponse { |
| // (Required) A reference provided by the SDK which represents a requests |
| // execution. The StateResponse must have the matching id when responding |
| // to the SDK. |
| string id = 1; |
| |
| // (Optional) If this is specified, then the state request has failed. |
| // A human readable string representing the reason as to why the request |
| // failed. |
| string error = 2; |
| |
| // (Optional) If this is specified, then the result of this state request |
| // can be cached using the supplied token. |
| bytes cache_token = 3; |
| |
| // A corresponding response matching the request will be populated. |
| oneof response { |
| // A response to getting state. |
| StateGetResponse get = 1000; |
| |
| // A response to appending to state. |
| StateAppendResponse append = 1001; |
| |
| // A response to clearing state. |
| StateClearResponse clear = 1002; |
| } |
| } |
| |
| service BeamFnState { |
| // Used to get/append/clear state stored by the runner on behalf of the SDK. |
| rpc State( |
| // A stream of state instructions requested of the runner. |
| stream StateRequest) |
| returns ( |
| // A stream of responses to state instructions the runner was asked to |
| // be performed. |
| stream StateResponse) {} |
| } |
| |
| message StateKey { |
| message Runner { |
| // (Required) Opaque information supplied by the runner. Used to support |
| // remote references. |
| bytes key = 1; |
| } |
| |
| message MultimapSideInput { |
| // (Required) The id of the PTransform containing a side input. |
| string ptransform_id = 1; |
| // (Required) The id of the side input. |
| string side_input_id = 2; |
| // (Required) The window (after mapping the currently executing elements |
| // window into the side input windows domain) encoded in a nested context. |
| bytes window = 3; |
| // (Required) The key encoded in a nested context. |
| bytes key = 4; |
| } |
| |
| message BagUserState { |
| // (Required) The id of the PTransform containing user state. |
| string ptransform_id = 1; |
| // (Required) The id of the user state. |
| string user_state_id = 2; |
| // (Required) The window encoded in a nested context. |
| bytes window = 3; |
| // (Required) The key of the currently executing element encoded in a |
| // nested context. |
| bytes key = 4; |
| } |
| |
| // (Required) One of the following state keys must be set. |
| oneof type { |
| Runner runner = 1; |
| MultimapSideInput multimap_side_input = 2; |
| BagUserState bag_user_state = 3; |
| // TODO: represent a state key for user map state |
| } |
| } |
| |
| // A request to get state. |
| message StateGetRequest { |
| // (Optional) If specified, signals to the runner that the response |
| // should resume from the following continuation token. |
| // |
| // If unspecified, signals to the runner that the response should start |
| // from the beginning of the logical continuable stream. |
| bytes continuation_token = 1; |
| } |
| |
| // A response to get state representing a logical byte stream which can be |
| // continued using the state API. |
| message StateGetResponse { |
| // (Optional) If specified, represents a token which can be used with the |
| // state API to get the next chunk of this logical byte stream. The end of |
| // the logical byte stream is signalled by this field being unset. |
| bytes continuation_token = 1; |
| |
| // Represents a part of a logical byte stream. Elements within |
| // the logical byte stream are encoded in the nested context and |
| // concatenated together. |
| bytes data = 2; |
| } |
| |
| // A request to append state. |
| message StateAppendRequest { |
| // Represents a part of a logical byte stream. Elements within |
| // the logical byte stream are encoded in the nested context and |
| // multiple append requests are concatenated together. |
| bytes data = 1; |
| } |
| |
| // A response to append state. |
| message StateAppendResponse {} |
| |
| // A request to clear state. |
| message StateClearRequest {} |
| |
| // A response to clear state. |
| message StateClearResponse {} |
| |
| /* |
| * Logging API |
| * |
| * This is very stable. There can be some changes to how we define a LogEntry, |
| * to increase/decrease the severity types, the way we format an exception/stack |
| * trace, or the log site. |
| */ |
| |
| // A log entry |
| message LogEntry { |
| // A list of log entries, enables buffering and batching of multiple |
| // log messages using the logging API. |
| message List { |
| // (Required) One or or more log messages. |
| repeated LogEntry log_entries = 1; |
| } |
| |
| // The severity of the event described in a log entry, expressed as one of the |
| // severity levels listed below. For your reference, the levels are |
| // assigned the listed numeric values. The effect of using numeric values |
| // other than those listed is undefined. |
| // |
| // If you are writing log entries, you should map other severity encodings to |
| // one of these standard levels. For example, you might map all of |
| // Java's FINE, FINER, and FINEST levels to `Severity.DEBUG`. |
| // |
| // This list is intentionally not comprehensive; the intent is to provide a |
| // common set of "good enough" severity levels so that logging front ends |
| // can provide filtering and searching across log types. Users of the API are |
| // free not to use all severity levels in their log messages. |
| message Severity { |
| enum Enum { |
| // Unspecified level information. Will be logged at the TRACE level. |
| UNSPECIFIED = 0; |
| TRACE = 1; |
| // Debugging information. |
| DEBUG = 2; |
| // Normal events. |
| INFO = 3; |
| // Normal but significant events, such as start up, shut down, or |
| // configuration. |
| NOTICE = 4; |
| // Warning events might cause problems. |
| WARN = 5; |
| // Error events are likely to cause problems. |
| ERROR = 6; |
| // Critical events cause severe problems or brief outages and may |
| // indicate that a person must take action. |
| CRITICAL = 7; |
| } |
| } |
| |
| // (Required) The severity of the log statement. |
| Severity.Enum severity = 1; |
| |
| // (Required) The time at which this log statement occurred. |
| google.protobuf.Timestamp timestamp = 2; |
| |
| // (Required) A human readable message. |
| string message = 3; |
| |
| // (Optional) An optional trace of the functions involved. For example, in |
| // Java this can include multiple causes and multiple suppressed exceptions. |
| string trace = 4; |
| |
| // (Optional) A reference to the instruction this log statement is associated |
| // with. |
| string instruction_reference = 5; |
| |
| // (Optional) A reference to the primitive transform this log statement is |
| // associated with. |
| string primitive_transform_reference = 6; |
| |
| // (Optional) Human-readable name of the function or method being invoked, |
| // with optional context such as the class or package name. The format can |
| // vary by language. For example: |
| // qual.if.ied.Class.method (Java) |
| // dir/package.func (Go) |
| // module.function (Python) |
| // file.cc:382 (C++) |
| string log_location = 7; |
| |
| // (Optional) The name of the thread this log statement is associated with. |
| string thread = 8; |
| } |
| |
| message LogControl {} |
| |
| // Stable |
| service BeamFnLogging { |
| // Allows for the SDK to emit log entries which the runner can |
| // associate with the active job. |
| rpc Logging( |
| // A stream of log entries batched into lists emitted by the SDK harness. |
| stream LogEntry.List) |
| returns ( |
| // A stream of log control messages used to configure the SDK. |
| stream LogControl) {} |
| } |
| |
| message NotifyRunnerAvailableRequest { |
| string worker_id = 1; |
| org.apache.beam.model.pipeline.v1.ApiServiceDescriptor control_endpoint = 2; |
| org.apache.beam.model.pipeline.v1.ApiServiceDescriptor logging_endpoint = 3; |
| org.apache.beam.model.pipeline.v1.ApiServiceDescriptor artifact_endpoint = 4; |
| org.apache.beam.model.pipeline.v1.ApiServiceDescriptor provision_endpoint = 5; |
| map<string, string> params = 10; |
| } |
| |
| message NotifyRunnerAvailableResponse { |
| string error = 1; |
| } |
| |
| service BeamFnExternalWorkerPool { |
| rpc NotifyRunnerAvailable(NotifyRunnerAvailableRequest) |
| returns (NotifyRunnerAvailableResponse) {} |
| } |