| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| """Generated message classes for dataflow version v1b3. |
| |
| Manages Google Cloud Dataflow projects on Google Cloud Platform. |
| """ |
| # NOTE: This file is autogenerated and should not be edited by hand. |
| |
| from __future__ import absolute_import |
| |
| from apitools.base.protorpclite import messages as _messages |
| from apitools.base.py import encoding |
| from apitools.base.py import extra_types |
| |
| |
| package = 'dataflow' |
| |
| |
| class ApproximateProgress(_messages.Message): |
| r"""Obsolete in favor of ApproximateReportedProgress and |
| ApproximateSplitRequest. |
| |
| Fields: |
| percentComplete: Obsolete. |
| position: Obsolete. |
| remainingTime: Obsolete. |
| """ |
| |
| percentComplete = _messages.FloatField(1, variant=_messages.Variant.FLOAT) |
| position = _messages.MessageField('Position', 2) |
| remainingTime = _messages.StringField(3) |
| |
| |
| class ApproximateReportedProgress(_messages.Message): |
| r"""A progress measurement of a WorkItem by a worker. |
| |
| Fields: |
| consumedParallelism: Total amount of parallelism in the portion of input |
| of this task that has already been consumed and is no longer active. In |
| the first two examples above (see remaining_parallelism), the value |
| should be 29 or 2 respectively. The sum of remaining_parallelism and |
| consumed_parallelism should equal the total amount of parallelism in |
| this work item. If specified, must be finite. |
| fractionConsumed: Completion as fraction of the input consumed, from 0.0 |
| (beginning, nothing consumed), to 1.0 (end of the input, entire input |
| consumed). |
| position: A Position within the work to represent a progress. |
| remainingParallelism: Total amount of parallelism in the input of this |
| task that remains, (i.e. can be delegated to this task and any new tasks |
| via dynamic splitting). Always at least 1 for non-finished work items |
| and 0 for finished. "Amount of parallelism" refers to how many non- |
| empty parts of the input can be read in parallel. This does not |
| necessarily equal number of records. An input that can be read in |
| parallel down to the individual records is called "perfectly |
| splittable". An example of non-perfectly parallelizable input is a |
| block-compressed file format where a block of records has to be read as |
| a whole, but different blocks can be read in parallel. Examples: * If |
| we are processing record #30 (starting at 1) out of 50 in a perfectly |
| splittable 50-record input, this value should be 21 (20 remaining + 1 |
| current). * If we are reading through block 3 in a block-compressed file |
| consisting of 5 blocks, this value should be 3 (since blocks 4 and 5 |
| can be processed in parallel by new tasks via dynamic splitting and |
| the current task remains processing block 3). * If we are reading |
| through the last block in a block-compressed file, or reading or |
| processing the last record in a perfectly splittable input, this value |
| should be 1, because apart from the current task, no additional |
| remainder can be split off. |
| """ |
| |
| consumedParallelism = _messages.MessageField('ReportedParallelism', 1) |
| fractionConsumed = _messages.FloatField(2) |
| position = _messages.MessageField('Position', 3) |
| remainingParallelism = _messages.MessageField('ReportedParallelism', 4) |
| |
| |
| class ApproximateSplitRequest(_messages.Message): |
| r"""A suggestion by the service to the worker to dynamically split the |
| WorkItem. |
| |
| Fields: |
| fractionConsumed: A fraction at which to split the work item, from 0.0 |
| (beginning of the input) to 1.0 (end of the input). |
| fractionOfRemainder: The fraction of the remainder of work to split the |
| work item at, from 0.0 (split at the current position) to 1.0 (end of |
| the input). |
| position: A Position at which to split the work item. |
| """ |
| |
| fractionConsumed = _messages.FloatField(1) |
| fractionOfRemainder = _messages.FloatField(2) |
| position = _messages.MessageField('Position', 3) |
| |
| |
| class AutoscalingEvent(_messages.Message): |
| r"""A structured message reporting an autoscaling decision made by the |
| Dataflow service. |
| |
| Enums: |
| EventTypeValueValuesEnum: The type of autoscaling event to report. |
| |
| Fields: |
| currentNumWorkers: The current number of workers the job has. |
| description: A message describing why the system decided to adjust the |
| current number of workers, why it failed, or why the system decided to |
| not make any changes to the number of workers. |
| eventType: The type of autoscaling event to report. |
| targetNumWorkers: The target number of workers the worker pool wants to |
| resize to use. |
| time: The time this event was emitted to indicate a new target or current |
| num_workers value. |
| workerPool: A short and friendly name for the worker pool this event |
| refers to, populated from the value of |
| PoolStageRelation::user_pool_name. |
| """ |
| |
| class EventTypeValueValuesEnum(_messages.Enum): |
| r"""The type of autoscaling event to report. |
| |
| Values: |
| TYPE_UNKNOWN: Default type for the enum. Value should never be |
| returned. |
| TARGET_NUM_WORKERS_CHANGED: The TARGET_NUM_WORKERS_CHANGED type should |
| be used when the target worker pool size has changed at the start of |
| an actuation. An event should always be specified as |
| TARGET_NUM_WORKERS_CHANGED if it reflects a change in the |
| target_num_workers. |
| CURRENT_NUM_WORKERS_CHANGED: The CURRENT_NUM_WORKERS_CHANGED type should |
| be used when actual worker pool size has been changed, but the |
| target_num_workers has not changed. |
| ACTUATION_FAILURE: The ACTUATION_FAILURE type should be used when we |
| want to report an error to the user indicating why the current number |
| of workers in the pool could not be changed. Displayed in the current |
| status and history widgets. |
| NO_CHANGE: Used when we want to report to the user a reason why we are |
| not currently adjusting the number of workers. Should specify both |
| target_num_workers, current_num_workers and a decision_message. |
| """ |
| TYPE_UNKNOWN = 0 |
| TARGET_NUM_WORKERS_CHANGED = 1 |
| CURRENT_NUM_WORKERS_CHANGED = 2 |
| ACTUATION_FAILURE = 3 |
| NO_CHANGE = 4 |
| |
| currentNumWorkers = _messages.IntegerField(1) |
| description = _messages.MessageField('StructuredMessage', 2) |
| eventType = _messages.EnumField('EventTypeValueValuesEnum', 3) |
| targetNumWorkers = _messages.IntegerField(4) |
| time = _messages.StringField(5) |
| workerPool = _messages.StringField(6) |
| |
| |
| class AutoscalingSettings(_messages.Message): |
| r"""Settings for WorkerPool autoscaling. |
| |
| Enums: |
| AlgorithmValueValuesEnum: The algorithm to use for autoscaling. |
| |
| Fields: |
| algorithm: The algorithm to use for autoscaling. |
| maxNumWorkers: The maximum number of workers to cap scaling at. |
| """ |
| |
| class AlgorithmValueValuesEnum(_messages.Enum): |
| r"""The algorithm to use for autoscaling. |
| |
| Values: |
| AUTOSCALING_ALGORITHM_UNKNOWN: The algorithm is unknown, or unspecified. |
| AUTOSCALING_ALGORITHM_NONE: Disable autoscaling. |
| AUTOSCALING_ALGORITHM_BASIC: Increase worker count over time to reduce |
| job execution time. |
| """ |
| AUTOSCALING_ALGORITHM_UNKNOWN = 0 |
| AUTOSCALING_ALGORITHM_NONE = 1 |
| AUTOSCALING_ALGORITHM_BASIC = 2 |
| |
| algorithm = _messages.EnumField('AlgorithmValueValuesEnum', 1) |
| maxNumWorkers = _messages.IntegerField(2, variant=_messages.Variant.INT32) |
| |
| |
| class BigQueryIODetails(_messages.Message): |
| r"""Metadata for a BigQuery connector used by the job. |
| |
| Fields: |
| dataset: Dataset accessed in the connection. |
| projectId: Project accessed in the connection. |
| query: Query used to access data in the connection. |
| table: Table accessed in the connection. |
| """ |
| |
| dataset = _messages.StringField(1) |
| projectId = _messages.StringField(2) |
| query = _messages.StringField(3) |
| table = _messages.StringField(4) |
| |
| |
| class BigTableIODetails(_messages.Message): |
| r"""Metadata for a BigTable connector used by the job. |
| |
| Fields: |
| instanceId: InstanceId accessed in the connection. |
| projectId: ProjectId accessed in the connection. |
| tableId: TableId accessed in the connection. |
| """ |
| |
| instanceId = _messages.StringField(1) |
| projectId = _messages.StringField(2) |
| tableId = _messages.StringField(3) |
| |
| |
| class CPUTime(_messages.Message): |
| r"""Modeled after information exposed by /proc/stat. |
| |
| Fields: |
| rate: Average CPU utilization rate (% non-idle cpu / second) since |
| previous sample. |
| timestamp: Timestamp of the measurement. |
| totalMs: Total active CPU time across all cores (ie., non-idle) in |
| milliseconds since start-up. |
| """ |
| |
| rate = _messages.FloatField(1) |
| timestamp = _messages.StringField(2) |
| totalMs = _messages.IntegerField(3, variant=_messages.Variant.UINT64) |
| |
| |
| class ComponentSource(_messages.Message): |
| r"""Description of an interstitial value between transforms in an execution |
| stage. |
| |
| Fields: |
| name: Dataflow service generated name for this source. |
| originalTransformOrCollection: User name for the original user transform |
| or collection with which this source is most closely associated. |
| userName: Human-readable name for this transform; may be user or system |
| generated. |
| """ |
| |
| name = _messages.StringField(1) |
| originalTransformOrCollection = _messages.StringField(2) |
| userName = _messages.StringField(3) |
| |
| |
| class ComponentTransform(_messages.Message): |
| r"""Description of a transform executed as part of an execution stage. |
| |
| Fields: |
| name: Dataflow service generated name for this source. |
| originalTransform: User name for the original user transform with which |
| this transform is most closely associated. |
| userName: Human-readable name for this transform; may be user or system |
| generated. |
| """ |
| |
| name = _messages.StringField(1) |
| originalTransform = _messages.StringField(2) |
| userName = _messages.StringField(3) |
| |
| |
| class ComputationTopology(_messages.Message): |
| r"""All configuration data for a particular Computation. |
| |
| Fields: |
| computationId: The ID of the computation. |
| inputs: The inputs to the computation. |
| keyRanges: The key ranges processed by the computation. |
| outputs: The outputs from the computation. |
| stateFamilies: The state family values. |
| systemStageName: The system stage name. |
| """ |
| |
| computationId = _messages.StringField(1) |
| inputs = _messages.MessageField('StreamLocation', 2, repeated=True) |
| keyRanges = _messages.MessageField('KeyRangeLocation', 3, repeated=True) |
| outputs = _messages.MessageField('StreamLocation', 4, repeated=True) |
| stateFamilies = _messages.MessageField('StateFamilyConfig', 5, repeated=True) |
| systemStageName = _messages.StringField(6) |
| |
| |
| class ConcatPosition(_messages.Message): |
| r"""A position that encapsulates an inner position and an index for the |
| inner position. A ConcatPosition can be used by a reader of a source that |
| encapsulates a set of other sources. |
| |
| Fields: |
| index: Index of the inner source. |
| position: Position within the inner source. |
| """ |
| |
| index = _messages.IntegerField(1, variant=_messages.Variant.INT32) |
| position = _messages.MessageField('Position', 2) |
| |
| |
| class CounterMetadata(_messages.Message): |
| r"""CounterMetadata includes all static non-name non-value counter |
| attributes. |
| |
| Enums: |
| KindValueValuesEnum: Counter aggregation kind. |
| StandardUnitsValueValuesEnum: System defined Units, see above enum. |
| |
| Fields: |
| description: Human-readable description of the counter semantics. |
| kind: Counter aggregation kind. |
| otherUnits: A string referring to the unit type. |
| standardUnits: System defined Units, see above enum. |
| """ |
| |
| class KindValueValuesEnum(_messages.Enum): |
| r"""Counter aggregation kind. |
| |
| Values: |
| INVALID: Counter aggregation kind was not set. |
| SUM: Aggregated value is the sum of all contributed values. |
| MAX: Aggregated value is the max of all contributed values. |
| MIN: Aggregated value is the min of all contributed values. |
| MEAN: Aggregated value is the mean of all contributed values. |
| OR: Aggregated value represents the logical 'or' of all contributed |
| values. |
| AND: Aggregated value represents the logical 'and' of all contributed |
| values. |
| SET: Aggregated value is a set of unique contributed values. |
| DISTRIBUTION: Aggregated value captures statistics about a distribution. |
| LATEST_VALUE: Aggregated value tracks the latest value of a variable. |
| """ |
| INVALID = 0 |
| SUM = 1 |
| MAX = 2 |
| MIN = 3 |
| MEAN = 4 |
| OR = 5 |
| AND = 6 |
| SET = 7 |
| DISTRIBUTION = 8 |
| LATEST_VALUE = 9 |
| |
| class StandardUnitsValueValuesEnum(_messages.Enum): |
| r"""System defined Units, see above enum. |
| |
| Values: |
| BYTES: Counter returns a value in bytes. |
| BYTES_PER_SEC: Counter returns a value in bytes per second. |
| MILLISECONDS: Counter returns a value in milliseconds. |
| MICROSECONDS: Counter returns a value in microseconds. |
| NANOSECONDS: Counter returns a value in nanoseconds. |
| TIMESTAMP_MSEC: Counter returns a timestamp in milliseconds. |
| TIMESTAMP_USEC: Counter returns a timestamp in microseconds. |
| TIMESTAMP_NSEC: Counter returns a timestamp in nanoseconds. |
| """ |
| BYTES = 0 |
| BYTES_PER_SEC = 1 |
| MILLISECONDS = 2 |
| MICROSECONDS = 3 |
| NANOSECONDS = 4 |
| TIMESTAMP_MSEC = 5 |
| TIMESTAMP_USEC = 6 |
| TIMESTAMP_NSEC = 7 |
| |
| description = _messages.StringField(1) |
| kind = _messages.EnumField('KindValueValuesEnum', 2) |
| otherUnits = _messages.StringField(3) |
| standardUnits = _messages.EnumField('StandardUnitsValueValuesEnum', 4) |
| |
| |
| class CounterStructuredName(_messages.Message): |
| r"""Identifies a counter within a per-job namespace. Counters whose |
| structured names are the same get merged into a single value for the job. |
| |
| Enums: |
| OriginValueValuesEnum: One of the standard Origins defined above. |
| PortionValueValuesEnum: Portion of this counter, either key or value. |
| |
| Fields: |
| componentStepName: Name of the optimized step being executed by the |
| workers. |
| executionStepName: Name of the stage. An execution step contains multiple |
| component steps. |
| inputIndex: Index of an input collection that's being read from/written to |
| as a side input. The index identifies a step's side inputs starting by 1 |
| (e.g. the first side input has input_index 1, the third has input_index |
| 3). Side inputs are identified by a pair of (original_step_name, |
| input_index). This field helps uniquely identify them. |
| name: Counter name. Not necessarily globally-unique, but unique within the |
| context of the other fields. Required. |
| origin: One of the standard Origins defined above. |
| originNamespace: A string containing a more specific namespace of the |
| counter's origin. |
| originalRequestingStepName: The step name requesting an operation, such as |
| GBK. I.e. the ParDo causing a read/write from shuffle to occur, or a |
| read from side inputs. |
| originalStepName: System generated name of the original step in the user's |
| graph, before optimization. |
| portion: Portion of this counter, either key or value. |
| workerId: ID of a particular worker. |
| """ |
| |
| class OriginValueValuesEnum(_messages.Enum): |
| r"""One of the standard Origins defined above. |
| |
| Values: |
| SYSTEM: Counter was created by the Dataflow system. |
| USER: Counter was created by the user. |
| """ |
| SYSTEM = 0 |
| USER = 1 |
| |
| class PortionValueValuesEnum(_messages.Enum): |
| r"""Portion of this counter, either key or value. |
| |
| Values: |
| ALL: Counter portion has not been set. |
| KEY: Counter reports a key. |
| VALUE: Counter reports a value. |
| """ |
| ALL = 0 |
| KEY = 1 |
| VALUE = 2 |
| |
| componentStepName = _messages.StringField(1) |
| executionStepName = _messages.StringField(2) |
| inputIndex = _messages.IntegerField(3, variant=_messages.Variant.INT32) |
| name = _messages.StringField(4) |
| origin = _messages.EnumField('OriginValueValuesEnum', 5) |
| originNamespace = _messages.StringField(6) |
| originalRequestingStepName = _messages.StringField(7) |
| originalStepName = _messages.StringField(8) |
| portion = _messages.EnumField('PortionValueValuesEnum', 9) |
| workerId = _messages.StringField(10) |
| |
| |
| class CounterStructuredNameAndMetadata(_messages.Message): |
| r"""A single message which encapsulates structured name and metadata for a |
| given counter. |
| |
| Fields: |
| metadata: Metadata associated with a counter |
| name: Structured name of the counter. |
| """ |
| |
| metadata = _messages.MessageField('CounterMetadata', 1) |
| name = _messages.MessageField('CounterStructuredName', 2) |
| |
| |
| class CounterUpdate(_messages.Message): |
| r"""An update to a Counter sent from a worker. |
| |
| Fields: |
| boolean: Boolean value for And, Or. |
| cumulative: True if this counter is reported as the total cumulative |
| aggregate value accumulated since the worker started working on this |
| WorkItem. By default this is false, indicating that this counter is |
| reported as a delta. |
| distribution: Distribution data |
| floatingPoint: Floating point value for Sum, Max, Min. |
| floatingPointList: List of floating point numbers, for Set. |
| floatingPointMean: Floating point mean aggregation value for Mean. |
| integer: Integer value for Sum, Max, Min. |
| integerGauge: Gauge data |
| integerList: List of integers, for Set. |
| integerMean: Integer mean aggregation value for Mean. |
| internal: Value for internally-defined counters used by the Dataflow |
| service. |
| nameAndKind: Counter name and aggregation type. |
| shortId: The service-generated short identifier for this counter. The |
| short_id -> (name, metadata) mapping is constant for the lifetime of a |
| job. |
| stringList: List of strings, for Set. |
| structuredNameAndMetadata: Counter structured name and metadata. |
| """ |
| |
| boolean = _messages.BooleanField(1) |
| cumulative = _messages.BooleanField(2) |
| distribution = _messages.MessageField('DistributionUpdate', 3) |
| floatingPoint = _messages.FloatField(4) |
| floatingPointList = _messages.MessageField('FloatingPointList', 5) |
| floatingPointMean = _messages.MessageField('FloatingPointMean', 6) |
| integer = _messages.MessageField('SplitInt64', 7) |
| integerGauge = _messages.MessageField('IntegerGauge', 8) |
| integerList = _messages.MessageField('IntegerList', 9) |
| integerMean = _messages.MessageField('IntegerMean', 10) |
| internal = _messages.MessageField('extra_types.JsonValue', 11) |
| nameAndKind = _messages.MessageField('NameAndKind', 12) |
| shortId = _messages.IntegerField(13) |
| stringList = _messages.MessageField('StringList', 14) |
| structuredNameAndMetadata = _messages.MessageField('CounterStructuredNameAndMetadata', 15) |
| |
| |
| class CreateJobFromTemplateRequest(_messages.Message): |
| r"""A request to create a Cloud Dataflow job from a template. |
| |
| Messages: |
| ParametersValue: The runtime parameters to pass to the job. |
| |
| Fields: |
| environment: The runtime environment for the job. |
| gcsPath: Required. A Cloud Storage path to the template from which to |
| create the job. Must be a valid Cloud Storage URL, beginning with |
| `gs://`. |
| jobName: Required. The job name to use for the created job. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to |
| which to direct the request. |
| parameters: The runtime parameters to pass to the job. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class ParametersValue(_messages.Message): |
| r"""The runtime parameters to pass to the job. |
| |
| Messages: |
| AdditionalProperty: An additional property for a ParametersValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type ParametersValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a ParametersValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| environment = _messages.MessageField('RuntimeEnvironment', 1) |
| gcsPath = _messages.StringField(2) |
| jobName = _messages.StringField(3) |
| location = _messages.StringField(4) |
| parameters = _messages.MessageField('ParametersValue', 5) |
| |
| |
| class CustomSourceLocation(_messages.Message): |
| r"""Identifies the location of a custom souce. |
| |
| Fields: |
| stateful: Whether this source is stateful. |
| """ |
| |
| stateful = _messages.BooleanField(1) |
| |
| |
| class DataDiskAssignment(_messages.Message): |
| r"""Data disk assignment for a given VM instance. |
| |
| Fields: |
| dataDisks: Mounted data disks. The order is important a data disk's |
| 0-based index in this list defines which persistent directory the disk |
| is mounted to, for example the list of { |
| "myproject-1014-104817-4c2-harness-0-disk-0" }, { |
| "myproject-1014-104817-4c2-harness-0-disk-1" }. |
| vmInstance: VM instance name the data disks mounted to, for example |
| "myproject-1014-104817-4c2-harness-0". |
| """ |
| |
| dataDisks = _messages.StringField(1, repeated=True) |
| vmInstance = _messages.StringField(2) |
| |
| |
| class DataflowProjectsDeleteSnapshotsRequest(_messages.Message): |
| r"""A DataflowProjectsDeleteSnapshotsRequest object. |
| |
| Fields: |
| location: The location that contains this snapshot. |
| projectId: The ID of the Cloud Platform project that the snapshot belongs |
| to. |
| snapshotId: The ID of the snapshot. |
| """ |
| |
| location = _messages.StringField(1) |
| projectId = _messages.StringField(2, required=True) |
| snapshotId = _messages.StringField(3) |
| |
| |
| class DataflowProjectsJobsAggregatedRequest(_messages.Message): |
| r"""A DataflowProjectsJobsAggregatedRequest object. |
| |
| Enums: |
| FilterValueValuesEnum: The kind of filter to use. |
| ViewValueValuesEnum: Level of information requested in response. Default |
| is `JOB_VIEW_SUMMARY`. |
| |
| Fields: |
| filter: The kind of filter to use. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| pageSize: If there are many jobs, limit response to at most this many. The |
| actual number of jobs returned will be the lesser of max_responses and |
| an unspecified server-defined limit. |
| pageToken: Set this to the 'next_page_token' field of a previous response |
| to request additional results in a long list. |
| projectId: The project which owns the jobs. |
| view: Level of information requested in response. Default is |
| `JOB_VIEW_SUMMARY`. |
| """ |
| |
| class FilterValueValuesEnum(_messages.Enum): |
| r"""The kind of filter to use. |
| |
| Values: |
| UNKNOWN: <no description> |
| ALL: <no description> |
| TERMINATED: <no description> |
| ACTIVE: <no description> |
| """ |
| UNKNOWN = 0 |
| ALL = 1 |
| TERMINATED = 2 |
| ACTIVE = 3 |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""Level of information requested in response. Default is |
| `JOB_VIEW_SUMMARY`. |
| |
| Values: |
| JOB_VIEW_UNKNOWN: <no description> |
| JOB_VIEW_SUMMARY: <no description> |
| JOB_VIEW_ALL: <no description> |
| JOB_VIEW_DESCRIPTION: <no description> |
| """ |
| JOB_VIEW_UNKNOWN = 0 |
| JOB_VIEW_SUMMARY = 1 |
| JOB_VIEW_ALL = 2 |
| JOB_VIEW_DESCRIPTION = 3 |
| |
| filter = _messages.EnumField('FilterValueValuesEnum', 1) |
| location = _messages.StringField(2) |
| pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32) |
| pageToken = _messages.StringField(4) |
| projectId = _messages.StringField(5, required=True) |
| view = _messages.EnumField('ViewValueValuesEnum', 6) |
| |
| |
| class DataflowProjectsJobsCreateRequest(_messages.Message): |
| r"""A DataflowProjectsJobsCreateRequest object. |
| |
| Enums: |
| ViewValueValuesEnum: The level of information requested in response. |
| |
| Fields: |
| job: A Job resource to be passed as the request body. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| projectId: The ID of the Cloud Platform project that the job belongs to. |
| replaceJobId: Deprecated. This field is now in the Job message. |
| view: The level of information requested in response. |
| """ |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""The level of information requested in response. |
| |
| Values: |
| JOB_VIEW_UNKNOWN: <no description> |
| JOB_VIEW_SUMMARY: <no description> |
| JOB_VIEW_ALL: <no description> |
| JOB_VIEW_DESCRIPTION: <no description> |
| """ |
| JOB_VIEW_UNKNOWN = 0 |
| JOB_VIEW_SUMMARY = 1 |
| JOB_VIEW_ALL = 2 |
| JOB_VIEW_DESCRIPTION = 3 |
| |
| job = _messages.MessageField('Job', 1) |
| location = _messages.StringField(2) |
| projectId = _messages.StringField(3, required=True) |
| replaceJobId = _messages.StringField(4) |
| view = _messages.EnumField('ViewValueValuesEnum', 5) |
| |
| |
| class DataflowProjectsJobsDebugGetConfigRequest(_messages.Message): |
| r"""A DataflowProjectsJobsDebugGetConfigRequest object. |
| |
| Fields: |
| getDebugConfigRequest: A GetDebugConfigRequest resource to be passed as |
| the request body. |
| jobId: The job id. |
| projectId: The project id. |
| """ |
| |
| getDebugConfigRequest = _messages.MessageField('GetDebugConfigRequest', 1) |
| jobId = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| |
| |
| class DataflowProjectsJobsDebugSendCaptureRequest(_messages.Message): |
| r"""A DataflowProjectsJobsDebugSendCaptureRequest object. |
| |
| Fields: |
| jobId: The job id. |
| projectId: The project id. |
| sendDebugCaptureRequest: A SendDebugCaptureRequest resource to be passed |
| as the request body. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| projectId = _messages.StringField(2, required=True) |
| sendDebugCaptureRequest = _messages.MessageField('SendDebugCaptureRequest', 3) |
| |
| |
| class DataflowProjectsJobsGetMetricsRequest(_messages.Message): |
| r"""A DataflowProjectsJobsGetMetricsRequest object. |
| |
| Fields: |
| jobId: The job to get messages for. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job specified by job_id. |
| projectId: A project id. |
| startTime: Return only metric data that has changed since this time. |
| Default is to return all information about all metrics for the job. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| location = _messages.StringField(2) |
| projectId = _messages.StringField(3, required=True) |
| startTime = _messages.StringField(4) |
| |
| |
| class DataflowProjectsJobsGetRequest(_messages.Message): |
| r"""A DataflowProjectsJobsGetRequest object. |
| |
| Enums: |
| ViewValueValuesEnum: The level of information requested in response. |
| |
| Fields: |
| jobId: The job ID. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| projectId: The ID of the Cloud Platform project that the job belongs to. |
| view: The level of information requested in response. |
| """ |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""The level of information requested in response. |
| |
| Values: |
| JOB_VIEW_UNKNOWN: <no description> |
| JOB_VIEW_SUMMARY: <no description> |
| JOB_VIEW_ALL: <no description> |
| JOB_VIEW_DESCRIPTION: <no description> |
| """ |
| JOB_VIEW_UNKNOWN = 0 |
| JOB_VIEW_SUMMARY = 1 |
| JOB_VIEW_ALL = 2 |
| JOB_VIEW_DESCRIPTION = 3 |
| |
| jobId = _messages.StringField(1, required=True) |
| location = _messages.StringField(2) |
| projectId = _messages.StringField(3, required=True) |
| view = _messages.EnumField('ViewValueValuesEnum', 4) |
| |
| |
| class DataflowProjectsJobsListRequest(_messages.Message): |
| r"""A DataflowProjectsJobsListRequest object. |
| |
| Enums: |
| FilterValueValuesEnum: The kind of filter to use. |
| ViewValueValuesEnum: Level of information requested in response. Default |
| is `JOB_VIEW_SUMMARY`. |
| |
| Fields: |
| filter: The kind of filter to use. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| pageSize: If there are many jobs, limit response to at most this many. The |
| actual number of jobs returned will be the lesser of max_responses and |
| an unspecified server-defined limit. |
| pageToken: Set this to the 'next_page_token' field of a previous response |
| to request additional results in a long list. |
| projectId: The project which owns the jobs. |
| view: Level of information requested in response. Default is |
| `JOB_VIEW_SUMMARY`. |
| """ |
| |
| class FilterValueValuesEnum(_messages.Enum): |
| r"""The kind of filter to use. |
| |
| Values: |
| UNKNOWN: <no description> |
| ALL: <no description> |
| TERMINATED: <no description> |
| ACTIVE: <no description> |
| """ |
| UNKNOWN = 0 |
| ALL = 1 |
| TERMINATED = 2 |
| ACTIVE = 3 |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""Level of information requested in response. Default is |
| `JOB_VIEW_SUMMARY`. |
| |
| Values: |
| JOB_VIEW_UNKNOWN: <no description> |
| JOB_VIEW_SUMMARY: <no description> |
| JOB_VIEW_ALL: <no description> |
| JOB_VIEW_DESCRIPTION: <no description> |
| """ |
| JOB_VIEW_UNKNOWN = 0 |
| JOB_VIEW_SUMMARY = 1 |
| JOB_VIEW_ALL = 2 |
| JOB_VIEW_DESCRIPTION = 3 |
| |
| filter = _messages.EnumField('FilterValueValuesEnum', 1) |
| location = _messages.StringField(2) |
| pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32) |
| pageToken = _messages.StringField(4) |
| projectId = _messages.StringField(5, required=True) |
| view = _messages.EnumField('ViewValueValuesEnum', 6) |
| |
| |
| class DataflowProjectsJobsMessagesListRequest(_messages.Message): |
| r"""A DataflowProjectsJobsMessagesListRequest object. |
| |
| Enums: |
| MinimumImportanceValueValuesEnum: Filter to only get messages with |
| importance >= level |
| |
| Fields: |
| endTime: Return only messages with timestamps < end_time. The default is |
| now (i.e. return up to the latest messages available). |
| jobId: The job to get messages about. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job specified by job_id. |
| minimumImportance: Filter to only get messages with importance >= level |
| pageSize: If specified, determines the maximum number of messages to |
| return. If unspecified, the service may choose an appropriate default, |
| or may return an arbitrarily large number of results. |
| pageToken: If supplied, this should be the value of next_page_token |
| returned by an earlier call. This will cause the next page of results to |
| be returned. |
| projectId: A project id. |
| startTime: If specified, return only messages with timestamps >= |
| start_time. The default is the job creation time (i.e. beginning of |
| messages). |
| """ |
| |
| class MinimumImportanceValueValuesEnum(_messages.Enum): |
| r"""Filter to only get messages with importance >= level |
| |
| Values: |
| JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description> |
| JOB_MESSAGE_DEBUG: <no description> |
| JOB_MESSAGE_DETAILED: <no description> |
| JOB_MESSAGE_BASIC: <no description> |
| JOB_MESSAGE_WARNING: <no description> |
| JOB_MESSAGE_ERROR: <no description> |
| """ |
| JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0 |
| JOB_MESSAGE_DEBUG = 1 |
| JOB_MESSAGE_DETAILED = 2 |
| JOB_MESSAGE_BASIC = 3 |
| JOB_MESSAGE_WARNING = 4 |
| JOB_MESSAGE_ERROR = 5 |
| |
| endTime = _messages.StringField(1) |
| jobId = _messages.StringField(2, required=True) |
| location = _messages.StringField(3) |
| minimumImportance = _messages.EnumField('MinimumImportanceValueValuesEnum', 4) |
| pageSize = _messages.IntegerField(5, variant=_messages.Variant.INT32) |
| pageToken = _messages.StringField(6) |
| projectId = _messages.StringField(7, required=True) |
| startTime = _messages.StringField(8) |
| |
| |
| class DataflowProjectsJobsSnapshotRequest(_messages.Message): |
| r"""A DataflowProjectsJobsSnapshotRequest object. |
| |
| Fields: |
| jobId: The job to be snapshotted. |
| projectId: The project which owns the job to be snapshotted. |
| snapshotJobRequest: A SnapshotJobRequest resource to be passed as the |
| request body. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| projectId = _messages.StringField(2, required=True) |
| snapshotJobRequest = _messages.MessageField('SnapshotJobRequest', 3) |
| |
| |
| class DataflowProjectsJobsUpdateRequest(_messages.Message): |
| r"""A DataflowProjectsJobsUpdateRequest object. |
| |
| Fields: |
| job: A Job resource to be passed as the request body. |
| jobId: The job ID. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| projectId: The ID of the Cloud Platform project that the job belongs to. |
| """ |
| |
| job = _messages.MessageField('Job', 1) |
| jobId = _messages.StringField(2, required=True) |
| location = _messages.StringField(3) |
| projectId = _messages.StringField(4, required=True) |
| |
| |
| class DataflowProjectsJobsWorkItemsLeaseRequest(_messages.Message): |
| r"""A DataflowProjectsJobsWorkItemsLeaseRequest object. |
| |
| Fields: |
| jobId: Identifies the workflow job this worker belongs to. |
| leaseWorkItemRequest: A LeaseWorkItemRequest resource to be passed as the |
| request body. |
| projectId: Identifies the project this worker belongs to. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| leaseWorkItemRequest = _messages.MessageField('LeaseWorkItemRequest', 2) |
| projectId = _messages.StringField(3, required=True) |
| |
| |
| class DataflowProjectsJobsWorkItemsReportStatusRequest(_messages.Message): |
| r"""A DataflowProjectsJobsWorkItemsReportStatusRequest object. |
| |
| Fields: |
| jobId: The job which the WorkItem is part of. |
| projectId: The project which owns the WorkItem's job. |
| reportWorkItemStatusRequest: A ReportWorkItemStatusRequest resource to be |
| passed as the request body. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| projectId = _messages.StringField(2, required=True) |
| reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 3) |
| |
| |
| class DataflowProjectsLocationsJobsCreateRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsCreateRequest object. |
| |
| Enums: |
| ViewValueValuesEnum: The level of information requested in response. |
| |
| Fields: |
| job: A Job resource to be passed as the request body. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| projectId: The ID of the Cloud Platform project that the job belongs to. |
| replaceJobId: Deprecated. This field is now in the Job message. |
| view: The level of information requested in response. |
| """ |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""The level of information requested in response. |
| |
| Values: |
| JOB_VIEW_UNKNOWN: <no description> |
| JOB_VIEW_SUMMARY: <no description> |
| JOB_VIEW_ALL: <no description> |
| JOB_VIEW_DESCRIPTION: <no description> |
| """ |
| JOB_VIEW_UNKNOWN = 0 |
| JOB_VIEW_SUMMARY = 1 |
| JOB_VIEW_ALL = 2 |
| JOB_VIEW_DESCRIPTION = 3 |
| |
| job = _messages.MessageField('Job', 1) |
| location = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| replaceJobId = _messages.StringField(4) |
| view = _messages.EnumField('ViewValueValuesEnum', 5) |
| |
| |
| class DataflowProjectsLocationsJobsDebugGetConfigRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsDebugGetConfigRequest object. |
| |
| Fields: |
| getDebugConfigRequest: A GetDebugConfigRequest resource to be passed as |
| the request body. |
| jobId: The job id. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job specified by job_id. |
| projectId: The project id. |
| """ |
| |
| getDebugConfigRequest = _messages.MessageField('GetDebugConfigRequest', 1) |
| jobId = _messages.StringField(2, required=True) |
| location = _messages.StringField(3, required=True) |
| projectId = _messages.StringField(4, required=True) |
| |
| |
| class DataflowProjectsLocationsJobsDebugSendCaptureRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsDebugSendCaptureRequest object. |
| |
| Fields: |
| jobId: The job id. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job specified by job_id. |
| projectId: The project id. |
| sendDebugCaptureRequest: A SendDebugCaptureRequest resource to be passed |
| as the request body. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| location = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| sendDebugCaptureRequest = _messages.MessageField('SendDebugCaptureRequest', 4) |
| |
| |
| class DataflowProjectsLocationsJobsGetMetricsRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsGetMetricsRequest object. |
| |
| Fields: |
| jobId: The job to get messages for. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job specified by job_id. |
| projectId: A project id. |
| startTime: Return only metric data that has changed since this time. |
| Default is to return all information about all metrics for the job. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| location = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| startTime = _messages.StringField(4) |
| |
| |
| class DataflowProjectsLocationsJobsGetRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsGetRequest object. |
| |
| Enums: |
| ViewValueValuesEnum: The level of information requested in response. |
| |
| Fields: |
| jobId: The job ID. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| projectId: The ID of the Cloud Platform project that the job belongs to. |
| view: The level of information requested in response. |
| """ |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""The level of information requested in response. |
| |
| Values: |
| JOB_VIEW_UNKNOWN: <no description> |
| JOB_VIEW_SUMMARY: <no description> |
| JOB_VIEW_ALL: <no description> |
| JOB_VIEW_DESCRIPTION: <no description> |
| """ |
| JOB_VIEW_UNKNOWN = 0 |
| JOB_VIEW_SUMMARY = 1 |
| JOB_VIEW_ALL = 2 |
| JOB_VIEW_DESCRIPTION = 3 |
| |
| jobId = _messages.StringField(1, required=True) |
| location = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| view = _messages.EnumField('ViewValueValuesEnum', 4) |
| |
| |
| class DataflowProjectsLocationsJobsListRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsListRequest object. |
| |
| Enums: |
| FilterValueValuesEnum: The kind of filter to use. |
| ViewValueValuesEnum: Level of information requested in response. Default |
| is `JOB_VIEW_SUMMARY`. |
| |
| Fields: |
| filter: The kind of filter to use. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| pageSize: If there are many jobs, limit response to at most this many. The |
| actual number of jobs returned will be the lesser of max_responses and |
| an unspecified server-defined limit. |
| pageToken: Set this to the 'next_page_token' field of a previous response |
| to request additional results in a long list. |
| projectId: The project which owns the jobs. |
| view: Level of information requested in response. Default is |
| `JOB_VIEW_SUMMARY`. |
| """ |
| |
| class FilterValueValuesEnum(_messages.Enum): |
| r"""The kind of filter to use. |
| |
| Values: |
| UNKNOWN: <no description> |
| ALL: <no description> |
| TERMINATED: <no description> |
| ACTIVE: <no description> |
| """ |
| UNKNOWN = 0 |
| ALL = 1 |
| TERMINATED = 2 |
| ACTIVE = 3 |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""Level of information requested in response. Default is |
| `JOB_VIEW_SUMMARY`. |
| |
| Values: |
| JOB_VIEW_UNKNOWN: <no description> |
| JOB_VIEW_SUMMARY: <no description> |
| JOB_VIEW_ALL: <no description> |
| JOB_VIEW_DESCRIPTION: <no description> |
| """ |
| JOB_VIEW_UNKNOWN = 0 |
| JOB_VIEW_SUMMARY = 1 |
| JOB_VIEW_ALL = 2 |
| JOB_VIEW_DESCRIPTION = 3 |
| |
| filter = _messages.EnumField('FilterValueValuesEnum', 1) |
| location = _messages.StringField(2, required=True) |
| pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32) |
| pageToken = _messages.StringField(4) |
| projectId = _messages.StringField(5, required=True) |
| view = _messages.EnumField('ViewValueValuesEnum', 6) |
| |
| |
| class DataflowProjectsLocationsJobsMessagesListRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsMessagesListRequest object. |
| |
| Enums: |
| MinimumImportanceValueValuesEnum: Filter to only get messages with |
| importance >= level |
| |
| Fields: |
| endTime: Return only messages with timestamps < end_time. The default is |
| now (i.e. return up to the latest messages available). |
| jobId: The job to get messages about. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job specified by job_id. |
| minimumImportance: Filter to only get messages with importance >= level |
| pageSize: If specified, determines the maximum number of messages to |
| return. If unspecified, the service may choose an appropriate default, |
| or may return an arbitrarily large number of results. |
| pageToken: If supplied, this should be the value of next_page_token |
| returned by an earlier call. This will cause the next page of results to |
| be returned. |
| projectId: A project id. |
| startTime: If specified, return only messages with timestamps >= |
| start_time. The default is the job creation time (i.e. beginning of |
| messages). |
| """ |
| |
| class MinimumImportanceValueValuesEnum(_messages.Enum): |
| r"""Filter to only get messages with importance >= level |
| |
| Values: |
| JOB_MESSAGE_IMPORTANCE_UNKNOWN: <no description> |
| JOB_MESSAGE_DEBUG: <no description> |
| JOB_MESSAGE_DETAILED: <no description> |
| JOB_MESSAGE_BASIC: <no description> |
| JOB_MESSAGE_WARNING: <no description> |
| JOB_MESSAGE_ERROR: <no description> |
| """ |
| JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0 |
| JOB_MESSAGE_DEBUG = 1 |
| JOB_MESSAGE_DETAILED = 2 |
| JOB_MESSAGE_BASIC = 3 |
| JOB_MESSAGE_WARNING = 4 |
| JOB_MESSAGE_ERROR = 5 |
| |
| endTime = _messages.StringField(1) |
| jobId = _messages.StringField(2, required=True) |
| location = _messages.StringField(3, required=True) |
| minimumImportance = _messages.EnumField('MinimumImportanceValueValuesEnum', 4) |
| pageSize = _messages.IntegerField(5, variant=_messages.Variant.INT32) |
| pageToken = _messages.StringField(6) |
| projectId = _messages.StringField(7, required=True) |
| startTime = _messages.StringField(8) |
| |
| |
| class DataflowProjectsLocationsJobsSnapshotRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsSnapshotRequest object. |
| |
| Fields: |
| jobId: The job to be snapshotted. |
| location: The location that contains this job. |
| projectId: The project which owns the job to be snapshotted. |
| snapshotJobRequest: A SnapshotJobRequest resource to be passed as the |
| request body. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| location = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| snapshotJobRequest = _messages.MessageField('SnapshotJobRequest', 4) |
| |
| |
| class DataflowProjectsLocationsJobsUpdateRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsUpdateRequest object. |
| |
| Fields: |
| job: A Job resource to be passed as the request body. |
| jobId: The job ID. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| projectId: The ID of the Cloud Platform project that the job belongs to. |
| """ |
| |
| job = _messages.MessageField('Job', 1) |
| jobId = _messages.StringField(2, required=True) |
| location = _messages.StringField(3, required=True) |
| projectId = _messages.StringField(4, required=True) |
| |
| |
| class DataflowProjectsLocationsJobsWorkItemsLeaseRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsWorkItemsLeaseRequest object. |
| |
| Fields: |
| jobId: Identifies the workflow job this worker belongs to. |
| leaseWorkItemRequest: A LeaseWorkItemRequest resource to be passed as the |
| request body. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the WorkItem's job. |
| projectId: Identifies the project this worker belongs to. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| leaseWorkItemRequest = _messages.MessageField('LeaseWorkItemRequest', 2) |
| location = _messages.StringField(3, required=True) |
| projectId = _messages.StringField(4, required=True) |
| |
| |
| class DataflowProjectsLocationsJobsWorkItemsReportStatusRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsJobsWorkItemsReportStatusRequest object. |
| |
| Fields: |
| jobId: The job which the WorkItem is part of. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the WorkItem's job. |
| projectId: The project which owns the WorkItem's job. |
| reportWorkItemStatusRequest: A ReportWorkItemStatusRequest resource to be |
| passed as the request body. |
| """ |
| |
| jobId = _messages.StringField(1, required=True) |
| location = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 4) |
| |
| |
| class DataflowProjectsLocationsSnapshotsDeleteRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsSnapshotsDeleteRequest object. |
| |
| Fields: |
| location: The location that contains this snapshot. |
| projectId: The ID of the Cloud Platform project that the snapshot belongs |
| to. |
| snapshotId: The ID of the snapshot. |
| """ |
| |
| location = _messages.StringField(1, required=True) |
| projectId = _messages.StringField(2, required=True) |
| snapshotId = _messages.StringField(3, required=True) |
| |
| |
| class DataflowProjectsLocationsSnapshotsGetRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsSnapshotsGetRequest object. |
| |
| Fields: |
| location: The location that contains this snapshot. |
| projectId: The ID of the Cloud Platform project that the snapshot belongs |
| to. |
| snapshotId: The ID of the snapshot. |
| """ |
| |
| location = _messages.StringField(1, required=True) |
| projectId = _messages.StringField(2, required=True) |
| snapshotId = _messages.StringField(3, required=True) |
| |
| |
| class DataflowProjectsLocationsSnapshotsListRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsSnapshotsListRequest object. |
| |
| Fields: |
| location: The location to list snapshots in. |
| projectId: The project ID to list snapshots for. |
| """ |
| |
| location = _messages.StringField(1, required=True) |
| projectId = _messages.StringField(2, required=True) |
| |
| |
| class DataflowProjectsLocationsSqlValidateRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsSqlValidateRequest object. |
| |
| Fields: |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to |
| which to direct the request. |
| projectId: Required. The ID of the Cloud Platform project that the job |
| belongs to. |
| query: The sql query to validate. |
| """ |
| |
| location = _messages.StringField(1, required=True) |
| projectId = _messages.StringField(2, required=True) |
| query = _messages.StringField(3) |
| |
| |
| class DataflowProjectsLocationsTemplatesCreateRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsTemplatesCreateRequest object. |
| |
| Fields: |
| createJobFromTemplateRequest: A CreateJobFromTemplateRequest resource to |
| be passed as the request body. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to |
| which to direct the request. |
| projectId: Required. The ID of the Cloud Platform project that the job |
| belongs to. |
| """ |
| |
| createJobFromTemplateRequest = _messages.MessageField('CreateJobFromTemplateRequest', 1) |
| location = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| |
| |
| class DataflowProjectsLocationsTemplatesGetRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsTemplatesGetRequest object. |
| |
| Enums: |
| ViewValueValuesEnum: The view to retrieve. Defaults to METADATA_ONLY. |
| |
| Fields: |
| gcsPath: Required. A Cloud Storage path to the template from which to |
| create the job. Must be valid Cloud Storage URL, beginning with 'gs://'. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to |
| which to direct the request. |
| projectId: Required. The ID of the Cloud Platform project that the job |
| belongs to. |
| view: The view to retrieve. Defaults to METADATA_ONLY. |
| """ |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""The view to retrieve. Defaults to METADATA_ONLY. |
| |
| Values: |
| METADATA_ONLY: <no description> |
| """ |
| METADATA_ONLY = 0 |
| |
| gcsPath = _messages.StringField(1) |
| location = _messages.StringField(2, required=True) |
| projectId = _messages.StringField(3, required=True) |
| view = _messages.EnumField('ViewValueValuesEnum', 4) |
| |
| |
| class DataflowProjectsLocationsTemplatesLaunchRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsTemplatesLaunchRequest object. |
| |
| Fields: |
| dynamicTemplate_gcsPath: Path to dynamic template spec file on GCS. The |
| file must be a Json serialized DynamicTemplateFieSpec object. |
| dynamicTemplate_stagingLocation: Cloud Storage path for staging |
| dependencies. Must be a valid Cloud Storage URL, beginning with `gs://`. |
| gcsPath: A Cloud Storage path to the template from which to create the |
| job. Must be valid Cloud Storage URL, beginning with 'gs://'. |
| launchTemplateParameters: A LaunchTemplateParameters resource to be passed |
| as the request body. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to |
| which to direct the request. |
| projectId: Required. The ID of the Cloud Platform project that the job |
| belongs to. |
| validateOnly: If true, the request is validated but not actually executed. |
| Defaults to false. |
| """ |
| |
| dynamicTemplate_gcsPath = _messages.StringField(1) |
| dynamicTemplate_stagingLocation = _messages.StringField(2) |
| gcsPath = _messages.StringField(3) |
| launchTemplateParameters = _messages.MessageField('LaunchTemplateParameters', 4) |
| location = _messages.StringField(5, required=True) |
| projectId = _messages.StringField(6, required=True) |
| validateOnly = _messages.BooleanField(7) |
| |
| |
| class DataflowProjectsLocationsWorkerMessagesRequest(_messages.Message): |
| r"""A DataflowProjectsLocationsWorkerMessagesRequest object. |
| |
| Fields: |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job. |
| projectId: The project to send the WorkerMessages to. |
| sendWorkerMessagesRequest: A SendWorkerMessagesRequest resource to be |
| passed as the request body. |
| """ |
| |
| location = _messages.StringField(1, required=True) |
| projectId = _messages.StringField(2, required=True) |
| sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 3) |
| |
| |
| class DataflowProjectsSnapshotsGetRequest(_messages.Message): |
| r"""A DataflowProjectsSnapshotsGetRequest object. |
| |
| Fields: |
| location: The location that contains this snapshot. |
| projectId: The ID of the Cloud Platform project that the snapshot belongs |
| to. |
| snapshotId: The ID of the snapshot. |
| """ |
| |
| location = _messages.StringField(1) |
| projectId = _messages.StringField(2, required=True) |
| snapshotId = _messages.StringField(3, required=True) |
| |
| |
| class DataflowProjectsSnapshotsListRequest(_messages.Message): |
| r"""A DataflowProjectsSnapshotsListRequest object. |
| |
| Fields: |
| location: The location to list snapshots in. |
| projectId: The project ID to list snapshots for. |
| """ |
| |
| location = _messages.StringField(1) |
| projectId = _messages.StringField(2, required=True) |
| |
| |
| class DataflowProjectsTemplatesCreateRequest(_messages.Message): |
| r"""A DataflowProjectsTemplatesCreateRequest object. |
| |
| Fields: |
| createJobFromTemplateRequest: A CreateJobFromTemplateRequest resource to |
| be passed as the request body. |
| projectId: Required. The ID of the Cloud Platform project that the job |
| belongs to. |
| """ |
| |
| createJobFromTemplateRequest = _messages.MessageField('CreateJobFromTemplateRequest', 1) |
| projectId = _messages.StringField(2, required=True) |
| |
| |
| class DataflowProjectsTemplatesGetRequest(_messages.Message): |
| r"""A DataflowProjectsTemplatesGetRequest object. |
| |
| Enums: |
| ViewValueValuesEnum: The view to retrieve. Defaults to METADATA_ONLY. |
| |
| Fields: |
| gcsPath: Required. A Cloud Storage path to the template from which to |
| create the job. Must be valid Cloud Storage URL, beginning with 'gs://'. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to |
| which to direct the request. |
| projectId: Required. The ID of the Cloud Platform project that the job |
| belongs to. |
| view: The view to retrieve. Defaults to METADATA_ONLY. |
| """ |
| |
| class ViewValueValuesEnum(_messages.Enum): |
| r"""The view to retrieve. Defaults to METADATA_ONLY. |
| |
| Values: |
| METADATA_ONLY: <no description> |
| """ |
| METADATA_ONLY = 0 |
| |
| gcsPath = _messages.StringField(1) |
| location = _messages.StringField(2) |
| projectId = _messages.StringField(3, required=True) |
| view = _messages.EnumField('ViewValueValuesEnum', 4) |
| |
| |
| class DataflowProjectsTemplatesLaunchRequest(_messages.Message): |
| r"""A DataflowProjectsTemplatesLaunchRequest object. |
| |
| Fields: |
| dynamicTemplate_gcsPath: Path to dynamic template spec file on GCS. The |
| file must be a Json serialized DynamicTemplateFieSpec object. |
| dynamicTemplate_stagingLocation: Cloud Storage path for staging |
| dependencies. Must be a valid Cloud Storage URL, beginning with `gs://`. |
| gcsPath: A Cloud Storage path to the template from which to create the |
| job. Must be valid Cloud Storage URL, beginning with 'gs://'. |
| launchTemplateParameters: A LaunchTemplateParameters resource to be passed |
| as the request body. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to |
| which to direct the request. |
| projectId: Required. The ID of the Cloud Platform project that the job |
| belongs to. |
| validateOnly: If true, the request is validated but not actually executed. |
| Defaults to false. |
| """ |
| |
| dynamicTemplate_gcsPath = _messages.StringField(1) |
| dynamicTemplate_stagingLocation = _messages.StringField(2) |
| gcsPath = _messages.StringField(3) |
| launchTemplateParameters = _messages.MessageField('LaunchTemplateParameters', 4) |
| location = _messages.StringField(5) |
| projectId = _messages.StringField(6, required=True) |
| validateOnly = _messages.BooleanField(7) |
| |
| |
| class DataflowProjectsWorkerMessagesRequest(_messages.Message): |
| r"""A DataflowProjectsWorkerMessagesRequest object. |
| |
| Fields: |
| projectId: The project to send the WorkerMessages to. |
| sendWorkerMessagesRequest: A SendWorkerMessagesRequest resource to be |
| passed as the request body. |
| """ |
| |
| projectId = _messages.StringField(1, required=True) |
| sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 2) |
| |
| |
| class DatastoreIODetails(_messages.Message): |
| r"""Metadata for a Datastore connector used by the job. |
| |
| Fields: |
| namespace: Namespace used in the connection. |
| projectId: ProjectId accessed in the connection. |
| """ |
| |
| namespace = _messages.StringField(1) |
| projectId = _messages.StringField(2) |
| |
| |
| class DeleteSnapshotResponse(_messages.Message): |
| r"""Response from deleting a snapshot.""" |
| |
| |
| class DerivedSource(_messages.Message): |
| r"""Specification of one of the bundles produced as a result of splitting a |
| Source (e.g. when executing a SourceSplitRequest, or when splitting an |
| active task using WorkItemStatus.dynamic_source_split), relative to the |
| source being split. |
| |
| Enums: |
| DerivationModeValueValuesEnum: What source to base the produced source on |
| (if any). |
| |
| Fields: |
| derivationMode: What source to base the produced source on (if any). |
| source: Specification of the source. |
| """ |
| |
| class DerivationModeValueValuesEnum(_messages.Enum): |
| r"""What source to base the produced source on (if any). |
| |
| Values: |
| SOURCE_DERIVATION_MODE_UNKNOWN: The source derivation is unknown, or |
| unspecified. |
| SOURCE_DERIVATION_MODE_INDEPENDENT: Produce a completely independent |
| Source with no base. |
| SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT: Produce a Source based on the |
| Source being split. |
| SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT: Produce a Source based on the |
| base of the Source being split. |
| """ |
| SOURCE_DERIVATION_MODE_UNKNOWN = 0 |
| SOURCE_DERIVATION_MODE_INDEPENDENT = 1 |
| SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT = 2 |
| SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT = 3 |
| |
| derivationMode = _messages.EnumField('DerivationModeValueValuesEnum', 1) |
| source = _messages.MessageField('Source', 2) |
| |
| |
| class Disk(_messages.Message): |
| r"""Describes the data disk used by a workflow job. |
| |
| Fields: |
| diskType: Disk storage type, as defined by Google Compute Engine. This |
| must be a disk type appropriate to the project and zone in which the |
| workers will run. If unknown or unspecified, the service will attempt |
| to choose a reasonable default. For example, the standard persistent |
| disk type is a resource name typically ending in "pd-standard". If SSD |
| persistent disks are available, the resource name typically ends with |
| "pd-ssd". The actual valid values are defined the Google Compute Engine |
| API, not by the Cloud Dataflow API; consult the Google Compute Engine |
| documentation for more information about determining the set of |
| available disk types for a particular project and zone. Google Compute |
| Engine Disk types are local to a particular project in a particular |
| zone, and so the resource name will typically look something like this: |
| compute.googleapis.com/projects/project-id/zones/zone/diskTypes/pd- |
| standard |
| mountPoint: Directory in a VM where disk is mounted. |
| sizeGb: Size of disk in GB. If zero or unspecified, the service will |
| attempt to choose a reasonable default. |
| """ |
| |
| diskType = _messages.StringField(1) |
| mountPoint = _messages.StringField(2) |
| sizeGb = _messages.IntegerField(3, variant=_messages.Variant.INT32) |
| |
| |
| class DisplayData(_messages.Message): |
| r"""Data provided with a pipeline or transform to provide descriptive info. |
| |
| Fields: |
| boolValue: Contains value if the data is of a boolean type. |
| durationValue: Contains value if the data is of duration type. |
| floatValue: Contains value if the data is of float type. |
| int64Value: Contains value if the data is of int64 type. |
| javaClassValue: Contains value if the data is of java class type. |
| key: The key identifying the display data. This is intended to be used as |
| a label for the display data when viewed in a dax monitoring system. |
| label: An optional label to display in a dax UI for the element. |
| namespace: The namespace for the key. This is usually a class name or |
| programming language namespace (i.e. python module) which defines the |
| display data. This allows a dax monitoring system to specially handle |
| the data and perform custom rendering. |
| shortStrValue: A possible additional shorter value to display. For example |
| a java_class_name_value of com.mypackage.MyDoFn will be stored with |
| MyDoFn as the short_str_value and com.mypackage.MyDoFn as the |
| java_class_name value. short_str_value can be displayed and |
| java_class_name_value will be displayed as a tooltip. |
| strValue: Contains value if the data is of string type. |
| timestampValue: Contains value if the data is of timestamp type. |
| url: An optional full URL. |
| """ |
| |
| boolValue = _messages.BooleanField(1) |
| durationValue = _messages.StringField(2) |
| floatValue = _messages.FloatField(3, variant=_messages.Variant.FLOAT) |
| int64Value = _messages.IntegerField(4) |
| javaClassValue = _messages.StringField(5) |
| key = _messages.StringField(6) |
| label = _messages.StringField(7) |
| namespace = _messages.StringField(8) |
| shortStrValue = _messages.StringField(9) |
| strValue = _messages.StringField(10) |
| timestampValue = _messages.StringField(11) |
| url = _messages.StringField(12) |
| |
| |
| class DistributionUpdate(_messages.Message): |
| r"""A metric value representing a distribution. |
| |
| Fields: |
| count: The count of the number of elements present in the distribution. |
| histogram: (Optional) Histogram of value counts for the distribution. |
| max: The maximum value present in the distribution. |
| min: The minimum value present in the distribution. |
| sum: Use an int64 since we'd prefer the added precision. If overflow is a |
| common problem we can detect it and use an additional int64 or a double. |
| sumOfSquares: Use a double since the sum of squares is likely to overflow |
| int64. |
| """ |
| |
| count = _messages.MessageField('SplitInt64', 1) |
| histogram = _messages.MessageField('Histogram', 2) |
| max = _messages.MessageField('SplitInt64', 3) |
| min = _messages.MessageField('SplitInt64', 4) |
| sum = _messages.MessageField('SplitInt64', 5) |
| sumOfSquares = _messages.FloatField(6) |
| |
| |
| class DynamicSourceSplit(_messages.Message): |
| r"""When a task splits using WorkItemStatus.dynamic_source_split, this |
| message describes the two parts of the split relative to the description of |
| the current task's input. |
| |
| Fields: |
| primary: Primary part (continued to be processed by worker). Specified |
| relative to the previously-current source. Becomes current. |
| residual: Residual part (returned to the pool of work). Specified relative |
| to the previously-current source. |
| """ |
| |
| primary = _messages.MessageField('DerivedSource', 1) |
| residual = _messages.MessageField('DerivedSource', 2) |
| |
| |
| class Environment(_messages.Message): |
| r"""Describes the environment in which a Dataflow Job runs. |
| |
| Enums: |
| FlexResourceSchedulingGoalValueValuesEnum: Which Flexible Resource |
| Scheduling mode to run in. |
| |
| Messages: |
| InternalExperimentsValue: Experimental settings. |
| SdkPipelineOptionsValue: The Cloud Dataflow SDK pipeline options specified |
| by the user. These options are passed through the service and are used |
| to recreate the SDK pipeline options on the worker in a language |
| agnostic and platform independent way. |
| UserAgentValue: A description of the process that generated the request. |
| VersionValue: A structure describing which components and their versions |
| of the service are required in order to run the job. |
| |
| Fields: |
| clusterManagerApiService: The type of cluster manager API to use. If |
| unknown or unspecified, the service will attempt to choose a reasonable |
| default. This should be in the form of the API service name, e.g. |
| "compute.googleapis.com". |
| dataset: The dataset for the current project where various workflow |
| related tables are stored. The supported resource type is: Google |
| BigQuery: bigquery.googleapis.com/{dataset} |
| experiments: The list of experiments to enable. |
| flexResourceSchedulingGoal: Which Flexible Resource Scheduling mode to run |
| in. |
| internalExperiments: Experimental settings. |
| sdkPipelineOptions: The Cloud Dataflow SDK pipeline options specified by |
| the user. These options are passed through the service and are used to |
| recreate the SDK pipeline options on the worker in a language agnostic |
| and platform independent way. |
| serviceAccountEmail: Identity to run virtual machines as. Defaults to the |
| default account. |
| serviceKmsKeyName: If set, contains the Cloud KMS key identifier used to |
| encrypt data at rest, AKA a Customer Managed Encryption Key (CMEK). |
| Format: |
| projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY |
| tempStoragePrefix: The prefix of the resources the system should use for |
| temporary storage. The system will append the suffix "/temp-{JOBNAME} |
| to this resource prefix, where {JOBNAME} is the value of the job_name |
| field. The resulting bucket and object prefix is used as the prefix of |
| the resources used to store temporary data needed during the job |
| execution. NOTE: This will override the value in taskrunner_settings. |
| The supported resource type is: Google Cloud Storage: |
| storage.googleapis.com/{bucket}/{object} |
| bucket.storage.googleapis.com/{object} |
| userAgent: A description of the process that generated the request. |
| version: A structure describing which components and their versions of the |
| service are required in order to run the job. |
| workerPools: The worker pools. At least one "harness" worker pool must be |
| specified in order for the job to have workers. |
| """ |
| |
| class FlexResourceSchedulingGoalValueValuesEnum(_messages.Enum): |
| r"""Which Flexible Resource Scheduling mode to run in. |
| |
| Values: |
| FLEXRS_UNSPECIFIED: Run in the default mode. |
| FLEXRS_SPEED_OPTIMIZED: Optimize for lower execution time. |
| FLEXRS_COST_OPTIMIZED: Optimize for lower cost. |
| """ |
| FLEXRS_UNSPECIFIED = 0 |
| FLEXRS_SPEED_OPTIMIZED = 1 |
| FLEXRS_COST_OPTIMIZED = 2 |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class InternalExperimentsValue(_messages.Message): |
| r"""Experimental settings. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| InternalExperimentsValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. Contains field @type |
| with type URL. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a InternalExperimentsValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class SdkPipelineOptionsValue(_messages.Message): |
| r"""The Cloud Dataflow SDK pipeline options specified by the user. These |
| options are passed through the service and are used to recreate the SDK |
| pipeline options on the worker in a language agnostic and platform |
| independent way. |
| |
| Messages: |
| AdditionalProperty: An additional property for a SdkPipelineOptionsValue |
| object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a SdkPipelineOptionsValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UserAgentValue(_messages.Message): |
| r"""A description of the process that generated the request. |
| |
| Messages: |
| AdditionalProperty: An additional property for a UserAgentValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UserAgentValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class VersionValue(_messages.Message): |
| r"""A structure describing which components and their versions of the |
| service are required in order to run the job. |
| |
| Messages: |
| AdditionalProperty: An additional property for a VersionValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a VersionValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| clusterManagerApiService = _messages.StringField(1) |
| dataset = _messages.StringField(2) |
| experiments = _messages.StringField(3, repeated=True) |
| flexResourceSchedulingGoal = _messages.EnumField('FlexResourceSchedulingGoalValueValuesEnum', 4) |
| internalExperiments = _messages.MessageField('InternalExperimentsValue', 5) |
| sdkPipelineOptions = _messages.MessageField('SdkPipelineOptionsValue', 6) |
| serviceAccountEmail = _messages.StringField(7) |
| serviceKmsKeyName = _messages.StringField(8) |
| tempStoragePrefix = _messages.StringField(9) |
| userAgent = _messages.MessageField('UserAgentValue', 10) |
| version = _messages.MessageField('VersionValue', 11) |
| workerPools = _messages.MessageField('WorkerPool', 12, repeated=True) |
| |
| |
| class ExecutionStageState(_messages.Message): |
| r"""A message describing the state of a particular execution stage. |
| |
| Enums: |
| ExecutionStageStateValueValuesEnum: Executions stage states allow the same |
| set of values as JobState. |
| |
| Fields: |
| currentStateTime: The time at which the stage transitioned to this state. |
| executionStageName: The name of the execution stage. |
| executionStageState: Executions stage states allow the same set of values |
| as JobState. |
| """ |
| |
| class ExecutionStageStateValueValuesEnum(_messages.Enum): |
| r"""Executions stage states allow the same set of values as JobState. |
| |
| Values: |
| JOB_STATE_UNKNOWN: The job's run state isn't specified. |
| JOB_STATE_STOPPED: `JOB_STATE_STOPPED` indicates that the job has not |
| yet started to run. |
| JOB_STATE_RUNNING: `JOB_STATE_RUNNING` indicates that the job is |
| currently running. |
| JOB_STATE_DONE: `JOB_STATE_DONE` indicates that the job has successfully |
| completed. This is a terminal job state. This state may be set by the |
| Cloud Dataflow service, as a transition from `JOB_STATE_RUNNING`. It |
| may also be set via a Cloud Dataflow `UpdateJob` call, if the job has |
| not yet reached a terminal state. |
| JOB_STATE_FAILED: `JOB_STATE_FAILED` indicates that the job has failed. |
| This is a terminal job state. This state may only be set by the Cloud |
| Dataflow service, and only as a transition from `JOB_STATE_RUNNING`. |
| JOB_STATE_CANCELLED: `JOB_STATE_CANCELLED` indicates that the job has |
| been explicitly cancelled. This is a terminal job state. This state |
| may only be set via a Cloud Dataflow `UpdateJob` call, and only if the |
| job has not yet reached another terminal state. |
| JOB_STATE_UPDATED: `JOB_STATE_UPDATED` indicates that the job was |
| successfully updated, meaning that this job was stopped and another |
| job was started, inheriting state from this one. This is a terminal |
| job state. This state may only be set by the Cloud Dataflow service, |
| and only as a transition from `JOB_STATE_RUNNING`. |
| JOB_STATE_DRAINING: `JOB_STATE_DRAINING` indicates that the job is in |
| the process of draining. A draining job has stopped pulling from its |
| input sources and is processing any data that remains in-flight. This |
| state may be set via a Cloud Dataflow `UpdateJob` call, but only as a |
| transition from `JOB_STATE_RUNNING`. Jobs that are draining may only |
| transition to `JOB_STATE_DRAINED`, `JOB_STATE_CANCELLED`, or |
| `JOB_STATE_FAILED`. |
| JOB_STATE_DRAINED: `JOB_STATE_DRAINED` indicates that the job has been |
| drained. A drained job terminated by stopping pulling from its input |
| sources and processing any data that remained in-flight when draining |
| was requested. This state is a terminal state, may only be set by the |
| Cloud Dataflow service, and only as a transition from |
| `JOB_STATE_DRAINING`. |
| JOB_STATE_PENDING: `JOB_STATE_PENDING` indicates that the job has been |
| created but is not yet running. Jobs that are pending may only |
| transition to `JOB_STATE_RUNNING`, or `JOB_STATE_FAILED`. |
| JOB_STATE_CANCELLING: `JOB_STATE_CANCELLING` indicates that the job has |
| been explicitly cancelled and is in the process of stopping. Jobs |
| that are cancelling may only transition to `JOB_STATE_CANCELLED` or |
| `JOB_STATE_FAILED`. |
| JOB_STATE_QUEUED: `JOB_STATE_QUEUED` indicates that the job has been |
| created but is being delayed until launch. Jobs that are queued may |
| only transition to `JOB_STATE_PENDING` or `JOB_STATE_CANCELLED`. |
| """ |
| JOB_STATE_UNKNOWN = 0 |
| JOB_STATE_STOPPED = 1 |
| JOB_STATE_RUNNING = 2 |
| JOB_STATE_DONE = 3 |
| JOB_STATE_FAILED = 4 |
| JOB_STATE_CANCELLED = 5 |
| JOB_STATE_UPDATED = 6 |
| JOB_STATE_DRAINING = 7 |
| JOB_STATE_DRAINED = 8 |
| JOB_STATE_PENDING = 9 |
| JOB_STATE_CANCELLING = 10 |
| JOB_STATE_QUEUED = 11 |
| |
| currentStateTime = _messages.StringField(1) |
| executionStageName = _messages.StringField(2) |
| executionStageState = _messages.EnumField('ExecutionStageStateValueValuesEnum', 3) |
| |
| |
| class ExecutionStageSummary(_messages.Message): |
| r"""Description of the composing transforms, names/ids, and input/outputs of |
| a stage of execution. Some composing transforms and sources may have been |
| generated by the Dataflow service during execution planning. |
| |
| Enums: |
| KindValueValuesEnum: Type of tranform this stage is executing. |
| |
| Fields: |
| componentSource: Collections produced and consumed by component transforms |
| of this stage. |
| componentTransform: Transforms that comprise this execution stage. |
| id: Dataflow service generated id for this stage. |
| inputSource: Input sources for this stage. |
| kind: Type of tranform this stage is executing. |
| name: Dataflow service generated name for this stage. |
| outputSource: Output sources for this stage. |
| """ |
| |
| class KindValueValuesEnum(_messages.Enum): |
| r"""Type of tranform this stage is executing. |
| |
| Values: |
| UNKNOWN_KIND: Unrecognized transform type. |
| PAR_DO_KIND: ParDo transform. |
| GROUP_BY_KEY_KIND: Group By Key transform. |
| FLATTEN_KIND: Flatten transform. |
| READ_KIND: Read transform. |
| WRITE_KIND: Write transform. |
| CONSTANT_KIND: Constructs from a constant value, such as with Create.of. |
| SINGLETON_KIND: Creates a Singleton view of a collection. |
| SHUFFLE_KIND: Opening or closing a shuffle session, often as part of a |
| GroupByKey. |
| """ |
| UNKNOWN_KIND = 0 |
| PAR_DO_KIND = 1 |
| GROUP_BY_KEY_KIND = 2 |
| FLATTEN_KIND = 3 |
| READ_KIND = 4 |
| WRITE_KIND = 5 |
| CONSTANT_KIND = 6 |
| SINGLETON_KIND = 7 |
| SHUFFLE_KIND = 8 |
| |
| componentSource = _messages.MessageField('ComponentSource', 1, repeated=True) |
| componentTransform = _messages.MessageField('ComponentTransform', 2, repeated=True) |
| id = _messages.StringField(3) |
| inputSource = _messages.MessageField('StageSource', 4, repeated=True) |
| kind = _messages.EnumField('KindValueValuesEnum', 5) |
| name = _messages.StringField(6) |
| outputSource = _messages.MessageField('StageSource', 7, repeated=True) |
| |
| |
| class FailedLocation(_messages.Message): |
| r"""Indicates which [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) failed |
| to respond to a request for data. |
| |
| Fields: |
| name: The name of the [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that failed to respond. |
| """ |
| |
| name = _messages.StringField(1) |
| |
| |
| class FileIODetails(_messages.Message): |
| r"""Metadata for a File connector used by the job. |
| |
| Fields: |
| filePattern: File Pattern used to access files by the connector. |
| """ |
| |
| filePattern = _messages.StringField(1) |
| |
| |
| class FlattenInstruction(_messages.Message): |
| r"""An instruction that copies its inputs (zero or more) to its (single) |
| output. |
| |
| Fields: |
| inputs: Describes the inputs to the flatten instruction. |
| """ |
| |
| inputs = _messages.MessageField('InstructionInput', 1, repeated=True) |
| |
| |
| class FloatingPointList(_messages.Message): |
| r"""A metric value representing a list of floating point numbers. |
| |
| Fields: |
| elements: Elements of the list. |
| """ |
| |
| elements = _messages.FloatField(1, repeated=True) |
| |
| |
| class FloatingPointMean(_messages.Message): |
| r"""A representation of a floating point mean metric contribution. |
| |
| Fields: |
| count: The number of values being aggregated. |
| sum: The sum of all values being aggregated. |
| """ |
| |
| count = _messages.MessageField('SplitInt64', 1) |
| sum = _messages.FloatField(2) |
| |
| |
| class GetDebugConfigRequest(_messages.Message): |
| r"""Request to get updated debug configuration for component. |
| |
| Fields: |
| componentId: The internal component id for which debug configuration is |
| requested. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job specified by job_id. |
| workerId: The worker id, i.e., VM hostname. |
| """ |
| |
| componentId = _messages.StringField(1) |
| location = _messages.StringField(2) |
| workerId = _messages.StringField(3) |
| |
| |
| class GetDebugConfigResponse(_messages.Message): |
| r"""Response to a get debug configuration request. |
| |
| Fields: |
| config: The encoded debug configuration for the requested component. |
| """ |
| |
| config = _messages.StringField(1) |
| |
| |
| class GetTemplateResponse(_messages.Message): |
| r"""The response to a GetTemplate request. |
| |
| Fields: |
| metadata: The template metadata describing the template name, available |
| parameters, etc. |
| status: The status of the get template request. Any problems with the |
| request will be indicated in the error_details. |
| """ |
| |
| metadata = _messages.MessageField('TemplateMetadata', 1) |
| status = _messages.MessageField('Status', 2) |
| |
| |
| class Histogram(_messages.Message): |
| r"""Histogram of value counts for a distribution. Buckets have an inclusive |
| lower bound and exclusive upper bound and use "1,2,5 bucketing": The first |
| bucket range is from [0,1) and all subsequent bucket boundaries are powers |
| of ten multiplied by 1, 2, or 5. Thus, bucket boundaries are 0, 1, 2, 5, 10, |
| 20, 50, 100, 200, 500, 1000, ... Negative values are not supported. |
| |
| Fields: |
| bucketCounts: Counts of values in each bucket. For efficiency, prefix and |
| trailing buckets with count = 0 are elided. Buckets can store the full |
| range of values of an unsigned long, with ULLONG_MAX falling into the |
| 59th bucket with range [1e19, 2e19). |
| firstBucketOffset: Starting index of first stored bucket. The non- |
| inclusive upper-bound of the ith bucket is given by: |
| pow(10,(i-first_bucket_offset)/3) * (1,2,5)[(i-first_bucket_offset)%3] |
| """ |
| |
| bucketCounts = _messages.IntegerField(1, repeated=True) |
| firstBucketOffset = _messages.IntegerField(2, variant=_messages.Variant.INT32) |
| |
| |
| class InstructionInput(_messages.Message): |
| r"""An input of an instruction, as a reference to an output of a producer |
| instruction. |
| |
| Fields: |
| outputNum: The output index (origin zero) within the producer. |
| producerInstructionIndex: The index (origin zero) of the parallel |
| instruction that produces the output to be consumed by this input. This |
| index is relative to the list of instructions in this input's |
| instruction's containing MapTask. |
| """ |
| |
| outputNum = _messages.IntegerField(1, variant=_messages.Variant.INT32) |
| producerInstructionIndex = _messages.IntegerField(2, variant=_messages.Variant.INT32) |
| |
| |
| class InstructionOutput(_messages.Message): |
| r"""An output of an instruction. |
| |
| Messages: |
| CodecValue: The codec to use to encode data being written via this output. |
| |
| Fields: |
| codec: The codec to use to encode data being written via this output. |
| name: The user-provided name of this output. |
| onlyCountKeyBytes: For system-generated byte and mean byte metrics, |
| certain instructions should only report the key size. |
| onlyCountValueBytes: For system-generated byte and mean byte metrics, |
| certain instructions should only report the value size. |
| originalName: System-defined name for this output in the original workflow |
| graph. Outputs that do not contribute to an original instruction do not |
| set this. |
| systemName: System-defined name of this output. Unique across the |
| workflow. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class CodecValue(_messages.Message): |
| r"""The codec to use to encode data being written via this output. |
| |
| Messages: |
| AdditionalProperty: An additional property for a CodecValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a CodecValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| codec = _messages.MessageField('CodecValue', 1) |
| name = _messages.StringField(2) |
| onlyCountKeyBytes = _messages.BooleanField(3) |
| onlyCountValueBytes = _messages.BooleanField(4) |
| originalName = _messages.StringField(5) |
| systemName = _messages.StringField(6) |
| |
| |
| class IntegerGauge(_messages.Message): |
| r"""A metric value representing temporal values of a variable. |
| |
| Fields: |
| timestamp: The time at which this value was measured. Measured as msecs |
| from epoch. |
| value: The value of the variable represented by this gauge. |
| """ |
| |
| timestamp = _messages.StringField(1) |
| value = _messages.MessageField('SplitInt64', 2) |
| |
| |
| class IntegerList(_messages.Message): |
| r"""A metric value representing a list of integers. |
| |
| Fields: |
| elements: Elements of the list. |
| """ |
| |
| elements = _messages.MessageField('SplitInt64', 1, repeated=True) |
| |
| |
| class IntegerMean(_messages.Message): |
| r"""A representation of an integer mean metric contribution. |
| |
| Fields: |
| count: The number of values being aggregated. |
| sum: The sum of all values being aggregated. |
| """ |
| |
| count = _messages.MessageField('SplitInt64', 1) |
| sum = _messages.MessageField('SplitInt64', 2) |
| |
| |
| class Job(_messages.Message): |
| r"""Defines a job to be run by the Cloud Dataflow service. |
| |
| Enums: |
| CurrentStateValueValuesEnum: The current state of the job. Jobs are |
| created in the `JOB_STATE_STOPPED` state unless otherwise specified. A |
| job in the `JOB_STATE_RUNNING` state may asynchronously enter a terminal |
| state. After a job has reached a terminal state, no further state |
| updates may be made. This field may be mutated by the Cloud Dataflow |
| service; callers cannot mutate it. |
| RequestedStateValueValuesEnum: The job's requested state. `UpdateJob` may |
| be used to switch between the `JOB_STATE_STOPPED` and |
| `JOB_STATE_RUNNING` states, by setting requested_state. `UpdateJob` may |
| also be used to directly set a job's requested state to |
| `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably terminating the |
| job if it has not already reached a terminal state. |
| TypeValueValuesEnum: The type of Cloud Dataflow job. |
| |
| Messages: |
| LabelsValue: User-defined labels for this job. The labels map can contain |
| no more than 64 entries. Entries of the labels map are UTF8 strings |
| that comply with the following restrictions: * Keys must conform to |
| regexp: \p{Ll}\p{Lo}{0,62} * Values must conform to regexp: |
| [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally |
| constrained to be <= 128 bytes in size. |
| TransformNameMappingValue: The map of transform name prefixes of the job |
| to be replaced to the corresponding name prefixes of the new job. |
| |
| Fields: |
| clientRequestId: The client's unique identifier of the job, re-used across |
| retried attempts. If this field is set, the service will ensure its |
| uniqueness. The request to create a job will fail if the service has |
| knowledge of a previously submitted job with the same client's ID and |
| job name. The caller may use this field to ensure idempotence of job |
| creation across retried attempts to create a job. By default, the field |
| is empty and, in that case, the service ignores it. |
| createTime: The timestamp when the job was initially created. Immutable |
| and set by the Cloud Dataflow service. |
| createdFromSnapshotId: If this is specified, the job's initial state is |
| populated from the given snapshot. |
| currentState: The current state of the job. Jobs are created in the |
| `JOB_STATE_STOPPED` state unless otherwise specified. A job in the |
| `JOB_STATE_RUNNING` state may asynchronously enter a terminal state. |
| After a job has reached a terminal state, no further state updates may |
| be made. This field may be mutated by the Cloud Dataflow service; |
| callers cannot mutate it. |
| currentStateTime: The timestamp associated with the current state. |
| environment: The environment for the job. |
| executionInfo: Deprecated. |
| id: The unique ID of this job. This field is set by the Cloud Dataflow |
| service when the Job is created, and is immutable for the life of the |
| job. |
| jobMetadata: This field is populated by the Dataflow service to support |
| filtering jobs by the metadata values provided here. Populated for |
| ListJobs and all GetJob views SUMMARY and higher. |
| labels: User-defined labels for this job. The labels map can contain no |
| more than 64 entries. Entries of the labels map are UTF8 strings that |
| comply with the following restrictions: * Keys must conform to regexp: |
| \p{Ll}\p{Lo}{0,62} * Values must conform to regexp: |
| [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally |
| constrained to be <= 128 bytes in size. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains this job. |
| name: The user-specified Cloud Dataflow job name. Only one Job with a |
| given name may exist in a project at any given time. If a caller |
| attempts to create a Job with the same name as an already-existing Job, |
| the attempt returns the existing Job. The name must match the regular |
| expression `[a-z]([-a-z0-9]{0,38}[a-z0-9])?` |
| pipelineDescription: Preliminary field: The format of this data may change |
| at any time. A description of the user pipeline and stages through which |
| it is executed. Created by Cloud Dataflow service. Only retrieved with |
| JOB_VIEW_DESCRIPTION or JOB_VIEW_ALL. |
| projectId: The ID of the Cloud Platform project that the job belongs to. |
| replaceJobId: If this job is an update of an existing job, this field is |
| the job ID of the job it replaced. When sending a `CreateJobRequest`, |
| you can update a job by specifying it here. The job named here is |
| stopped, and its intermediate state is transferred to this job. |
| replacedByJobId: If another job is an update of this job (and thus, this |
| job is in `JOB_STATE_UPDATED`), this field contains the ID of that job. |
| requestedState: The job's requested state. `UpdateJob` may be used to |
| switch between the `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING` states, |
| by setting requested_state. `UpdateJob` may also be used to directly |
| set a job's requested state to `JOB_STATE_CANCELLED` or |
| `JOB_STATE_DONE`, irrevocably terminating the job if it has not already |
| reached a terminal state. |
| stageStates: This field may be mutated by the Cloud Dataflow service; |
| callers cannot mutate it. |
| startTime: The timestamp when the job was started (transitioned to |
| JOB_STATE_PENDING). Flexible resource scheduling jobs are started with |
| some delay after job creation, so start_time is unset before start and |
| is updated when the job is started by the Cloud Dataflow service. For |
| other jobs, start_time always equals to create_time and is immutable and |
| set by the Cloud Dataflow service. |
| steps: Exactly one of step or steps_location should be specified. The |
| top-level steps that constitute the entire job. |
| stepsLocation: The GCS location where the steps are stored. |
| tempFiles: A set of files the system should be aware of that are used for |
| temporary storage. These temporary files will be removed on job |
| completion. No duplicates are allowed. No file patterns are supported. |
| The supported files are: Google Cloud Storage: |
| storage.googleapis.com/{bucket}/{object} |
| bucket.storage.googleapis.com/{object} |
| transformNameMapping: The map of transform name prefixes of the job to be |
| replaced to the corresponding name prefixes of the new job. |
| type: The type of Cloud Dataflow job. |
| """ |
| |
| class CurrentStateValueValuesEnum(_messages.Enum): |
| r"""The current state of the job. Jobs are created in the |
| `JOB_STATE_STOPPED` state unless otherwise specified. A job in the |
| `JOB_STATE_RUNNING` state may asynchronously enter a terminal state. After |
| a job has reached a terminal state, no further state updates may be made. |
| This field may be mutated by the Cloud Dataflow service; callers cannot |
| mutate it. |
| |
| Values: |
| JOB_STATE_UNKNOWN: The job's run state isn't specified. |
| JOB_STATE_STOPPED: `JOB_STATE_STOPPED` indicates that the job has not |
| yet started to run. |
| JOB_STATE_RUNNING: `JOB_STATE_RUNNING` indicates that the job is |
| currently running. |
| JOB_STATE_DONE: `JOB_STATE_DONE` indicates that the job has successfully |
| completed. This is a terminal job state. This state may be set by the |
| Cloud Dataflow service, as a transition from `JOB_STATE_RUNNING`. It |
| may also be set via a Cloud Dataflow `UpdateJob` call, if the job has |
| not yet reached a terminal state. |
| JOB_STATE_FAILED: `JOB_STATE_FAILED` indicates that the job has failed. |
| This is a terminal job state. This state may only be set by the Cloud |
| Dataflow service, and only as a transition from `JOB_STATE_RUNNING`. |
| JOB_STATE_CANCELLED: `JOB_STATE_CANCELLED` indicates that the job has |
| been explicitly cancelled. This is a terminal job state. This state |
| may only be set via a Cloud Dataflow `UpdateJob` call, and only if the |
| job has not yet reached another terminal state. |
| JOB_STATE_UPDATED: `JOB_STATE_UPDATED` indicates that the job was |
| successfully updated, meaning that this job was stopped and another |
| job was started, inheriting state from this one. This is a terminal |
| job state. This state may only be set by the Cloud Dataflow service, |
| and only as a transition from `JOB_STATE_RUNNING`. |
| JOB_STATE_DRAINING: `JOB_STATE_DRAINING` indicates that the job is in |
| the process of draining. A draining job has stopped pulling from its |
| input sources and is processing any data that remains in-flight. This |
| state may be set via a Cloud Dataflow `UpdateJob` call, but only as a |
| transition from `JOB_STATE_RUNNING`. Jobs that are draining may only |
| transition to `JOB_STATE_DRAINED`, `JOB_STATE_CANCELLED`, or |
| `JOB_STATE_FAILED`. |
| JOB_STATE_DRAINED: `JOB_STATE_DRAINED` indicates that the job has been |
| drained. A drained job terminated by stopping pulling from its input |
| sources and processing any data that remained in-flight when draining |
| was requested. This state is a terminal state, may only be set by the |
| Cloud Dataflow service, and only as a transition from |
| `JOB_STATE_DRAINING`. |
| JOB_STATE_PENDING: `JOB_STATE_PENDING` indicates that the job has been |
| created but is not yet running. Jobs that are pending may only |
| transition to `JOB_STATE_RUNNING`, or `JOB_STATE_FAILED`. |
| JOB_STATE_CANCELLING: `JOB_STATE_CANCELLING` indicates that the job has |
| been explicitly cancelled and is in the process of stopping. Jobs |
| that are cancelling may only transition to `JOB_STATE_CANCELLED` or |
| `JOB_STATE_FAILED`. |
| JOB_STATE_QUEUED: `JOB_STATE_QUEUED` indicates that the job has been |
| created but is being delayed until launch. Jobs that are queued may |
| only transition to `JOB_STATE_PENDING` or `JOB_STATE_CANCELLED`. |
| """ |
| JOB_STATE_UNKNOWN = 0 |
| JOB_STATE_STOPPED = 1 |
| JOB_STATE_RUNNING = 2 |
| JOB_STATE_DONE = 3 |
| JOB_STATE_FAILED = 4 |
| JOB_STATE_CANCELLED = 5 |
| JOB_STATE_UPDATED = 6 |
| JOB_STATE_DRAINING = 7 |
| JOB_STATE_DRAINED = 8 |
| JOB_STATE_PENDING = 9 |
| JOB_STATE_CANCELLING = 10 |
| JOB_STATE_QUEUED = 11 |
| |
| class RequestedStateValueValuesEnum(_messages.Enum): |
| r"""The job's requested state. `UpdateJob` may be used to switch between |
| the `JOB_STATE_STOPPED` and `JOB_STATE_RUNNING` states, by setting |
| requested_state. `UpdateJob` may also be used to directly set a job's |
| requested state to `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably |
| terminating the job if it has not already reached a terminal state. |
| |
| Values: |
| JOB_STATE_UNKNOWN: The job's run state isn't specified. |
| JOB_STATE_STOPPED: `JOB_STATE_STOPPED` indicates that the job has not |
| yet started to run. |
| JOB_STATE_RUNNING: `JOB_STATE_RUNNING` indicates that the job is |
| currently running. |
| JOB_STATE_DONE: `JOB_STATE_DONE` indicates that the job has successfully |
| completed. This is a terminal job state. This state may be set by the |
| Cloud Dataflow service, as a transition from `JOB_STATE_RUNNING`. It |
| may also be set via a Cloud Dataflow `UpdateJob` call, if the job has |
| not yet reached a terminal state. |
| JOB_STATE_FAILED: `JOB_STATE_FAILED` indicates that the job has failed. |
| This is a terminal job state. This state may only be set by the Cloud |
| Dataflow service, and only as a transition from `JOB_STATE_RUNNING`. |
| JOB_STATE_CANCELLED: `JOB_STATE_CANCELLED` indicates that the job has |
| been explicitly cancelled. This is a terminal job state. This state |
| may only be set via a Cloud Dataflow `UpdateJob` call, and only if the |
| job has not yet reached another terminal state. |
| JOB_STATE_UPDATED: `JOB_STATE_UPDATED` indicates that the job was |
| successfully updated, meaning that this job was stopped and another |
| job was started, inheriting state from this one. This is a terminal |
| job state. This state may only be set by the Cloud Dataflow service, |
| and only as a transition from `JOB_STATE_RUNNING`. |
| JOB_STATE_DRAINING: `JOB_STATE_DRAINING` indicates that the job is in |
| the process of draining. A draining job has stopped pulling from its |
| input sources and is processing any data that remains in-flight. This |
| state may be set via a Cloud Dataflow `UpdateJob` call, but only as a |
| transition from `JOB_STATE_RUNNING`. Jobs that are draining may only |
| transition to `JOB_STATE_DRAINED`, `JOB_STATE_CANCELLED`, or |
| `JOB_STATE_FAILED`. |
| JOB_STATE_DRAINED: `JOB_STATE_DRAINED` indicates that the job has been |
| drained. A drained job terminated by stopping pulling from its input |
| sources and processing any data that remained in-flight when draining |
| was requested. This state is a terminal state, may only be set by the |
| Cloud Dataflow service, and only as a transition from |
| `JOB_STATE_DRAINING`. |
| JOB_STATE_PENDING: `JOB_STATE_PENDING` indicates that the job has been |
| created but is not yet running. Jobs that are pending may only |
| transition to `JOB_STATE_RUNNING`, or `JOB_STATE_FAILED`. |
| JOB_STATE_CANCELLING: `JOB_STATE_CANCELLING` indicates that the job has |
| been explicitly cancelled and is in the process of stopping. Jobs |
| that are cancelling may only transition to `JOB_STATE_CANCELLED` or |
| `JOB_STATE_FAILED`. |
| JOB_STATE_QUEUED: `JOB_STATE_QUEUED` indicates that the job has been |
| created but is being delayed until launch. Jobs that are queued may |
| only transition to `JOB_STATE_PENDING` or `JOB_STATE_CANCELLED`. |
| """ |
| JOB_STATE_UNKNOWN = 0 |
| JOB_STATE_STOPPED = 1 |
| JOB_STATE_RUNNING = 2 |
| JOB_STATE_DONE = 3 |
| JOB_STATE_FAILED = 4 |
| JOB_STATE_CANCELLED = 5 |
| JOB_STATE_UPDATED = 6 |
| JOB_STATE_DRAINING = 7 |
| JOB_STATE_DRAINED = 8 |
| JOB_STATE_PENDING = 9 |
| JOB_STATE_CANCELLING = 10 |
| JOB_STATE_QUEUED = 11 |
| |
| class TypeValueValuesEnum(_messages.Enum): |
| r"""The type of Cloud Dataflow job. |
| |
| Values: |
| JOB_TYPE_UNKNOWN: The type of the job is unspecified, or unknown. |
| JOB_TYPE_BATCH: A batch job with a well-defined end point: data is read, |
| data is processed, data is written, and the job is done. |
| JOB_TYPE_STREAMING: A continuously streaming job with no end: data is |
| read, processed, and written continuously. |
| """ |
| JOB_TYPE_UNKNOWN = 0 |
| JOB_TYPE_BATCH = 1 |
| JOB_TYPE_STREAMING = 2 |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class LabelsValue(_messages.Message): |
| r"""User-defined labels for this job. The labels map can contain no more |
| than 64 entries. Entries of the labels map are UTF8 strings that comply |
| with the following restrictions: * Keys must conform to regexp: |
| \p{Ll}\p{Lo}{0,62} * Values must conform to regexp: |
| [\p{Ll}\p{Lo}\p{N}_-]{0,63} * Both keys and values are additionally |
| constrained to be <= 128 bytes in size. |
| |
| Messages: |
| AdditionalProperty: An additional property for a LabelsValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type LabelsValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a LabelsValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class TransformNameMappingValue(_messages.Message): |
| r"""The map of transform name prefixes of the job to be replaced to the |
| corresponding name prefixes of the new job. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| TransformNameMappingValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type |
| TransformNameMappingValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a TransformNameMappingValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| clientRequestId = _messages.StringField(1) |
| createTime = _messages.StringField(2) |
| createdFromSnapshotId = _messages.StringField(3) |
| currentState = _messages.EnumField('CurrentStateValueValuesEnum', 4) |
| currentStateTime = _messages.StringField(5) |
| environment = _messages.MessageField('Environment', 6) |
| executionInfo = _messages.MessageField('JobExecutionInfo', 7) |
| id = _messages.StringField(8) |
| jobMetadata = _messages.MessageField('JobMetadata', 9) |
| labels = _messages.MessageField('LabelsValue', 10) |
| location = _messages.StringField(11) |
| name = _messages.StringField(12) |
| pipelineDescription = _messages.MessageField('PipelineDescription', 13) |
| projectId = _messages.StringField(14) |
| replaceJobId = _messages.StringField(15) |
| replacedByJobId = _messages.StringField(16) |
| requestedState = _messages.EnumField('RequestedStateValueValuesEnum', 17) |
| stageStates = _messages.MessageField('ExecutionStageState', 18, repeated=True) |
| startTime = _messages.StringField(19) |
| steps = _messages.MessageField('Step', 20, repeated=True) |
| stepsLocation = _messages.StringField(21) |
| tempFiles = _messages.StringField(22, repeated=True) |
| transformNameMapping = _messages.MessageField('TransformNameMappingValue', 23) |
| type = _messages.EnumField('TypeValueValuesEnum', 24) |
| |
| |
| class JobExecutionInfo(_messages.Message): |
| r"""Additional information about how a Cloud Dataflow job will be executed |
| that isn't contained in the submitted job. |
| |
| Messages: |
| StagesValue: A mapping from each stage to the information about that |
| stage. |
| |
| Fields: |
| stages: A mapping from each stage to the information about that stage. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class StagesValue(_messages.Message): |
| r"""A mapping from each stage to the information about that stage. |
| |
| Messages: |
| AdditionalProperty: An additional property for a StagesValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type StagesValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a StagesValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A JobExecutionStageInfo attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('JobExecutionStageInfo', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| stages = _messages.MessageField('StagesValue', 1) |
| |
| |
| class JobExecutionStageInfo(_messages.Message): |
| r"""Contains information about how a particular google.dataflow.v1beta3.Step |
| will be executed. |
| |
| Fields: |
| stepName: The steps associated with the execution stage. Note that stages |
| may have several steps, and that a given step might be run by more than |
| one stage. |
| """ |
| |
| stepName = _messages.StringField(1, repeated=True) |
| |
| |
| class JobMessage(_messages.Message): |
| r"""A particular message pertaining to a Dataflow job. |
| |
| Enums: |
| MessageImportanceValueValuesEnum: Importance level of the message. |
| |
| Fields: |
| id: Deprecated. |
| messageImportance: Importance level of the message. |
| messageText: The text of the message. |
| time: The timestamp of the message. |
| """ |
| |
| class MessageImportanceValueValuesEnum(_messages.Enum): |
| r"""Importance level of the message. |
| |
| Values: |
| JOB_MESSAGE_IMPORTANCE_UNKNOWN: The message importance isn't specified, |
| or is unknown. |
| JOB_MESSAGE_DEBUG: The message is at the 'debug' level: typically only |
| useful for software engineers working on the code the job is running. |
| Typically, Dataflow pipeline runners do not display log messages at |
| this level by default. |
| JOB_MESSAGE_DETAILED: The message is at the 'detailed' level: somewhat |
| verbose, but potentially useful to users. Typically, Dataflow |
| pipeline runners do not display log messages at this level by default. |
| These messages are displayed by default in the Dataflow monitoring UI. |
| JOB_MESSAGE_BASIC: The message is at the 'basic' level: useful for |
| keeping track of the execution of a Dataflow pipeline. Typically, |
| Dataflow pipeline runners display log messages at this level by |
| default, and these messages are displayed by default in the Dataflow |
| monitoring UI. |
| JOB_MESSAGE_WARNING: The message is at the 'warning' level: indicating a |
| condition pertaining to a job which may require human intervention. |
| Typically, Dataflow pipeline runners display log messages at this |
| level by default, and these messages are displayed by default in the |
| Dataflow monitoring UI. |
| JOB_MESSAGE_ERROR: The message is at the 'error' level: indicating a |
| condition preventing a job from succeeding. Typically, Dataflow |
| pipeline runners display log messages at this level by default, and |
| these messages are displayed by default in the Dataflow monitoring UI. |
| """ |
| JOB_MESSAGE_IMPORTANCE_UNKNOWN = 0 |
| JOB_MESSAGE_DEBUG = 1 |
| JOB_MESSAGE_DETAILED = 2 |
| JOB_MESSAGE_BASIC = 3 |
| JOB_MESSAGE_WARNING = 4 |
| JOB_MESSAGE_ERROR = 5 |
| |
| id = _messages.StringField(1) |
| messageImportance = _messages.EnumField('MessageImportanceValueValuesEnum', 2) |
| messageText = _messages.StringField(3) |
| time = _messages.StringField(4) |
| |
| |
| class JobMetadata(_messages.Message): |
| r"""Metadata available primarily for filtering jobs. Will be included in the |
| ListJob response and Job SUMMARY view. |
| |
| Fields: |
| bigTableDetails: Identification of a BigTable source used in the Dataflow |
| job. |
| bigqueryDetails: Identification of a BigQuery source used in the Dataflow |
| job. |
| datastoreDetails: Identification of a Datastore source used in the |
| Dataflow job. |
| fileDetails: Identification of a File source used in the Dataflow job. |
| pubsubDetails: Identification of a PubSub source used in the Dataflow job. |
| sdkVersion: The SDK version used to run the job. |
| spannerDetails: Identification of a Spanner source used in the Dataflow |
| job. |
| """ |
| |
| bigTableDetails = _messages.MessageField('BigTableIODetails', 1, repeated=True) |
| bigqueryDetails = _messages.MessageField('BigQueryIODetails', 2, repeated=True) |
| datastoreDetails = _messages.MessageField('DatastoreIODetails', 3, repeated=True) |
| fileDetails = _messages.MessageField('FileIODetails', 4, repeated=True) |
| pubsubDetails = _messages.MessageField('PubSubIODetails', 5, repeated=True) |
| sdkVersion = _messages.MessageField('SdkVersion', 6) |
| spannerDetails = _messages.MessageField('SpannerIODetails', 7, repeated=True) |
| |
| |
| class JobMetrics(_messages.Message): |
| r"""JobMetrics contains a collection of metrics describing the detailed |
| progress of a Dataflow job. Metrics correspond to user-defined and system- |
| defined metrics in the job. This resource captures only the most recent |
| values of each metric; time-series data can be queried for them (under the |
| same metric names) from Cloud Monitoring. |
| |
| Fields: |
| metricTime: Timestamp as of which metric values are current. |
| metrics: All metrics for this job. |
| """ |
| |
| metricTime = _messages.StringField(1) |
| metrics = _messages.MessageField('MetricUpdate', 2, repeated=True) |
| |
| |
| class KeyRangeDataDiskAssignment(_messages.Message): |
| r"""Data disk assignment information for a specific key-range of a sharded |
| computation. Currently we only support UTF-8 character splits to simplify |
| encoding into JSON. |
| |
| Fields: |
| dataDisk: The name of the data disk where data for this range is stored. |
| This name is local to the Google Cloud Platform project and uniquely |
| identifies the disk within that project, for example |
| "myproject-1014-104817-4c2-harness-0-disk-1". |
| end: The end (exclusive) of the key range. |
| start: The start (inclusive) of the key range. |
| """ |
| |
| dataDisk = _messages.StringField(1) |
| end = _messages.StringField(2) |
| start = _messages.StringField(3) |
| |
| |
| class KeyRangeLocation(_messages.Message): |
| r"""Location information for a specific key-range of a sharded computation. |
| Currently we only support UTF-8 character splits to simplify encoding into |
| JSON. |
| |
| Fields: |
| dataDisk: The name of the data disk where data for this range is stored. |
| This name is local to the Google Cloud Platform project and uniquely |
| identifies the disk within that project, for example |
| "myproject-1014-104817-4c2-harness-0-disk-1". |
| deliveryEndpoint: The physical location of this range assignment to be |
| used for streaming computation cross-worker message delivery. |
| deprecatedPersistentDirectory: DEPRECATED. The location of the persistent |
| state for this range, as a persistent directory in the worker local |
| filesystem. |
| end: The end (exclusive) of the key range. |
| start: The start (inclusive) of the key range. |
| """ |
| |
| dataDisk = _messages.StringField(1) |
| deliveryEndpoint = _messages.StringField(2) |
| deprecatedPersistentDirectory = _messages.StringField(3) |
| end = _messages.StringField(4) |
| start = _messages.StringField(5) |
| |
| |
| class LaunchTemplateParameters(_messages.Message): |
| r"""Parameters to provide to the template being launched. |
| |
| Messages: |
| ParametersValue: The runtime parameters to pass to the job. |
| |
| Fields: |
| environment: The runtime environment for the job. |
| jobName: Required. The job name to use for the created job. |
| parameters: The runtime parameters to pass to the job. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class ParametersValue(_messages.Message): |
| r"""The runtime parameters to pass to the job. |
| |
| Messages: |
| AdditionalProperty: An additional property for a ParametersValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type ParametersValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a ParametersValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| environment = _messages.MessageField('RuntimeEnvironment', 1) |
| jobName = _messages.StringField(2) |
| parameters = _messages.MessageField('ParametersValue', 3) |
| |
| |
| class LaunchTemplateResponse(_messages.Message): |
| r"""Response to the request to launch a template. |
| |
| Fields: |
| job: The job that was launched, if the request was not a dry run and the |
| job was successfully launched. |
| """ |
| |
| job = _messages.MessageField('Job', 1) |
| |
| |
| class LeaseWorkItemRequest(_messages.Message): |
| r"""Request to lease WorkItems. |
| |
| Messages: |
| UnifiedWorkerRequestValue: Untranslated bag-of-bytes WorkRequest from |
| UnifiedWorker. |
| |
| Fields: |
| currentWorkerTime: The current timestamp at the worker. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the WorkItem's job. |
| requestedLeaseDuration: The initial lease period. |
| unifiedWorkerRequest: Untranslated bag-of-bytes WorkRequest from |
| UnifiedWorker. |
| workItemTypes: Filter for WorkItem type. |
| workerCapabilities: Worker capabilities. WorkItems might be limited to |
| workers with specific capabilities. |
| workerId: Identifies the worker leasing work -- typically the ID of the |
| virtual machine running the worker. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UnifiedWorkerRequestValue(_messages.Message): |
| r"""Untranslated bag-of-bytes WorkRequest from UnifiedWorker. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| UnifiedWorkerRequestValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. Contains field @type |
| with type URL. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UnifiedWorkerRequestValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| currentWorkerTime = _messages.StringField(1) |
| location = _messages.StringField(2) |
| requestedLeaseDuration = _messages.StringField(3) |
| unifiedWorkerRequest = _messages.MessageField('UnifiedWorkerRequestValue', 4) |
| workItemTypes = _messages.StringField(5, repeated=True) |
| workerCapabilities = _messages.StringField(6, repeated=True) |
| workerId = _messages.StringField(7) |
| |
| |
| class LeaseWorkItemResponse(_messages.Message): |
| r"""Response to a request to lease WorkItems. |
| |
| Messages: |
| UnifiedWorkerResponseValue: Untranslated bag-of-bytes WorkResponse for |
| UnifiedWorker. |
| |
| Fields: |
| unifiedWorkerResponse: Untranslated bag-of-bytes WorkResponse for |
| UnifiedWorker. |
| workItems: A list of the leased WorkItems. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UnifiedWorkerResponseValue(_messages.Message): |
| r"""Untranslated bag-of-bytes WorkResponse for UnifiedWorker. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| UnifiedWorkerResponseValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. Contains field @type |
| with type URL. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UnifiedWorkerResponseValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| unifiedWorkerResponse = _messages.MessageField('UnifiedWorkerResponseValue', 1) |
| workItems = _messages.MessageField('WorkItem', 2, repeated=True) |
| |
| |
| class ListJobMessagesResponse(_messages.Message): |
| r"""Response to a request to list job messages. |
| |
| Fields: |
| autoscalingEvents: Autoscaling events in ascending timestamp order. |
| jobMessages: Messages in ascending timestamp order. |
| nextPageToken: The token to obtain the next page of results if there are |
| more. |
| """ |
| |
| autoscalingEvents = _messages.MessageField('AutoscalingEvent', 1, repeated=True) |
| jobMessages = _messages.MessageField('JobMessage', 2, repeated=True) |
| nextPageToken = _messages.StringField(3) |
| |
| |
| class ListJobsResponse(_messages.Message): |
| r"""Response to a request to list Cloud Dataflow jobs. This may be a |
| partial response, depending on the page size in the ListJobsRequest. |
| |
| Fields: |
| failedLocation: Zero or more messages describing the [regional endpoints] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that failed to respond. |
| jobs: A subset of the requested job information. |
| nextPageToken: Set if there may be more results than fit in this response. |
| """ |
| |
| failedLocation = _messages.MessageField('FailedLocation', 1, repeated=True) |
| jobs = _messages.MessageField('Job', 2, repeated=True) |
| nextPageToken = _messages.StringField(3) |
| |
| |
| class ListSnapshotsResponse(_messages.Message): |
| r"""List of snapshots. |
| |
| Fields: |
| snapshots: Returned snapshots. |
| """ |
| |
| snapshots = _messages.MessageField('Snapshot', 1, repeated=True) |
| |
| |
| class MapTask(_messages.Message): |
| r"""MapTask consists of an ordered set of instructions, each of which |
| describes one particular low-level operation for the worker to perform in |
| order to accomplish the MapTask's WorkItem. Each instruction must appear in |
| the list before any instructions which depends on its output. |
| |
| Fields: |
| counterPrefix: Counter prefix that can be used to prefix counters. Not |
| currently used in Dataflow. |
| instructions: The instructions in the MapTask. |
| stageName: System-defined name of the stage containing this MapTask. |
| Unique across the workflow. |
| systemName: System-defined name of this MapTask. Unique across the |
| workflow. |
| """ |
| |
| counterPrefix = _messages.StringField(1) |
| instructions = _messages.MessageField('ParallelInstruction', 2, repeated=True) |
| stageName = _messages.StringField(3) |
| systemName = _messages.StringField(4) |
| |
| |
| class MetricShortId(_messages.Message): |
| r"""The metric short id is returned to the user alongside an offset into |
| ReportWorkItemStatusRequest |
| |
| Fields: |
| metricIndex: The index of the corresponding metric in the |
| ReportWorkItemStatusRequest. Required. |
| shortId: The service-generated short identifier for the metric. |
| """ |
| |
| metricIndex = _messages.IntegerField(1, variant=_messages.Variant.INT32) |
| shortId = _messages.IntegerField(2) |
| |
| |
| class MetricStructuredName(_messages.Message): |
| r"""Identifies a metric, by describing the source which generated the |
| metric. |
| |
| Messages: |
| ContextValue: Zero or more labeled fields which identify the part of the |
| job this metric is associated with, such as the name of a step or |
| collection. For example, built-in counters associated with steps will |
| have context['step'] = <step-name>. Counters associated with |
| PCollections in the SDK will have context['pcollection'] = <pcollection- |
| name>. |
| |
| Fields: |
| context: Zero or more labeled fields which identify the part of the job |
| this metric is associated with, such as the name of a step or |
| collection. For example, built-in counters associated with steps will |
| have context['step'] = <step-name>. Counters associated with |
| PCollections in the SDK will have context['pcollection'] = <pcollection- |
| name>. |
| name: Worker-defined metric name. |
| origin: Origin (namespace) of metric name. May be blank for user-define |
| metrics; will be "dataflow" for metrics defined by the Dataflow service |
| or SDK. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class ContextValue(_messages.Message): |
| r"""Zero or more labeled fields which identify the part of the job this |
| metric is associated with, such as the name of a step or collection. For |
| example, built-in counters associated with steps will have context['step'] |
| = <step-name>. Counters associated with PCollections in the SDK will have |
| context['pcollection'] = <pcollection-name>. |
| |
| Messages: |
| AdditionalProperty: An additional property for a ContextValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type ContextValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a ContextValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| context = _messages.MessageField('ContextValue', 1) |
| name = _messages.StringField(2) |
| origin = _messages.StringField(3) |
| |
| |
| class MetricUpdate(_messages.Message): |
| r"""Describes the state of a metric. |
| |
| Fields: |
| cumulative: True if this metric is reported as the total cumulative |
| aggregate value accumulated since the worker started working on this |
| WorkItem. By default this is false, indicating that this metric is |
| reported as a delta that is not associated with any WorkItem. |
| distribution: A struct value describing properties of a distribution of |
| numeric values. |
| gauge: A struct value describing properties of a Gauge. Metrics of gauge |
| type show the value of a metric across time, and is aggregated based on |
| the newest value. |
| internal: Worker-computed aggregate value for internal use by the Dataflow |
| service. |
| kind: Metric aggregation kind. The possible metric aggregation kinds are |
| "Sum", "Max", "Min", "Mean", "Set", "And", "Or", and "Distribution". The |
| specified aggregation kind is case-insensitive. If omitted, this is not |
| an aggregated value but instead a single metric sample value. |
| meanCount: Worker-computed aggregate value for the "Mean" aggregation |
| kind. This holds the count of the aggregated values and is used in |
| combination with mean_sum above to obtain the actual mean aggregate |
| value. The only possible value type is Long. |
| meanSum: Worker-computed aggregate value for the "Mean" aggregation kind. |
| This holds the sum of the aggregated values and is used in combination |
| with mean_count below to obtain the actual mean aggregate value. The |
| only possible value types are Long and Double. |
| name: Name of the metric. |
| scalar: Worker-computed aggregate value for aggregation kinds "Sum", |
| "Max", "Min", "And", and "Or". The possible value types are Long, |
| Double, and Boolean. |
| set: Worker-computed aggregate value for the "Set" aggregation kind. The |
| only possible value type is a list of Values whose type can be Long, |
| Double, or String, according to the metric's type. All Values in the |
| list must be of the same type. |
| updateTime: Timestamp associated with the metric value. Optional when |
| workers are reporting work progress; it will be filled in responses from |
| the metrics API. |
| """ |
| |
| cumulative = _messages.BooleanField(1) |
| distribution = _messages.MessageField('extra_types.JsonValue', 2) |
| gauge = _messages.MessageField('extra_types.JsonValue', 3) |
| internal = _messages.MessageField('extra_types.JsonValue', 4) |
| kind = _messages.StringField(5) |
| meanCount = _messages.MessageField('extra_types.JsonValue', 6) |
| meanSum = _messages.MessageField('extra_types.JsonValue', 7) |
| name = _messages.MessageField('MetricStructuredName', 8) |
| scalar = _messages.MessageField('extra_types.JsonValue', 9) |
| set = _messages.MessageField('extra_types.JsonValue', 10) |
| updateTime = _messages.StringField(11) |
| |
| |
| class MountedDataDisk(_messages.Message): |
| r"""Describes mounted data disk. |
| |
| Fields: |
| dataDisk: The name of the data disk. This name is local to the Google |
| Cloud Platform project and uniquely identifies the disk within that |
| project, for example "myproject-1014-104817-4c2-harness-0-disk-1". |
| """ |
| |
| dataDisk = _messages.StringField(1) |
| |
| |
| class MultiOutputInfo(_messages.Message): |
| r"""Information about an output of a multi-output DoFn. |
| |
| Fields: |
| tag: The id of the tag the user code will emit to this output by; this |
| should correspond to the tag of some SideInputInfo. |
| """ |
| |
| tag = _messages.StringField(1) |
| |
| |
| class NameAndKind(_messages.Message): |
| r"""Basic metadata about a counter. |
| |
| Enums: |
| KindValueValuesEnum: Counter aggregation kind. |
| |
| Fields: |
| kind: Counter aggregation kind. |
| name: Name of the counter. |
| """ |
| |
| class KindValueValuesEnum(_messages.Enum): |
| r"""Counter aggregation kind. |
| |
| Values: |
| INVALID: Counter aggregation kind was not set. |
| SUM: Aggregated value is the sum of all contributed values. |
| MAX: Aggregated value is the max of all contributed values. |
| MIN: Aggregated value is the min of all contributed values. |
| MEAN: Aggregated value is the mean of all contributed values. |
| OR: Aggregated value represents the logical 'or' of all contributed |
| values. |
| AND: Aggregated value represents the logical 'and' of all contributed |
| values. |
| SET: Aggregated value is a set of unique contributed values. |
| DISTRIBUTION: Aggregated value captures statistics about a distribution. |
| LATEST_VALUE: Aggregated value tracks the latest value of a variable. |
| """ |
| INVALID = 0 |
| SUM = 1 |
| MAX = 2 |
| MIN = 3 |
| MEAN = 4 |
| OR = 5 |
| AND = 6 |
| SET = 7 |
| DISTRIBUTION = 8 |
| LATEST_VALUE = 9 |
| |
| kind = _messages.EnumField('KindValueValuesEnum', 1) |
| name = _messages.StringField(2) |
| |
| |
| class Package(_messages.Message): |
| r"""The packages that must be installed in order for a worker to run the |
| steps of the Cloud Dataflow job that will be assigned to its worker pool. |
| This is the mechanism by which the Cloud Dataflow SDK causes code to be |
| loaded onto the workers. For example, the Cloud Dataflow Java SDK might use |
| this to install jars containing the user's code and all of the various |
| dependencies (libraries, data files, etc.) required in order for that code |
| to run. |
| |
| Fields: |
| location: The resource to read the package from. The supported resource |
| type is: Google Cloud Storage: storage.googleapis.com/{bucket} |
| bucket.storage.googleapis.com/ |
| name: The name of the package. |
| """ |
| |
| location = _messages.StringField(1) |
| name = _messages.StringField(2) |
| |
| |
| class ParDoInstruction(_messages.Message): |
| r"""An instruction that does a ParDo operation. Takes one main input and |
| zero or more side inputs, and produces zero or more outputs. Runs user code. |
| |
| Messages: |
| UserFnValue: The user function to invoke. |
| |
| Fields: |
| input: The input. |
| multiOutputInfos: Information about each of the outputs, if user_fn is a |
| MultiDoFn. |
| numOutputs: The number of outputs. |
| sideInputs: Zero or more side inputs. |
| userFn: The user function to invoke. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UserFnValue(_messages.Message): |
| r"""The user function to invoke. |
| |
| Messages: |
| AdditionalProperty: An additional property for a UserFnValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UserFnValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| input = _messages.MessageField('InstructionInput', 1) |
| multiOutputInfos = _messages.MessageField('MultiOutputInfo', 2, repeated=True) |
| numOutputs = _messages.IntegerField(3, variant=_messages.Variant.INT32) |
| sideInputs = _messages.MessageField('SideInputInfo', 4, repeated=True) |
| userFn = _messages.MessageField('UserFnValue', 5) |
| |
| |
| class ParallelInstruction(_messages.Message): |
| r"""Describes a particular operation comprising a MapTask. |
| |
| Fields: |
| flatten: Additional information for Flatten instructions. |
| name: User-provided name of this operation. |
| originalName: System-defined name for the operation in the original |
| workflow graph. |
| outputs: Describes the outputs of the instruction. |
| parDo: Additional information for ParDo instructions. |
| partialGroupByKey: Additional information for PartialGroupByKey |
| instructions. |
| read: Additional information for Read instructions. |
| systemName: System-defined name of this operation. Unique across the |
| workflow. |
| write: Additional information for Write instructions. |
| """ |
| |
| flatten = _messages.MessageField('FlattenInstruction', 1) |
| name = _messages.StringField(2) |
| originalName = _messages.StringField(3) |
| outputs = _messages.MessageField('InstructionOutput', 4, repeated=True) |
| parDo = _messages.MessageField('ParDoInstruction', 5) |
| partialGroupByKey = _messages.MessageField('PartialGroupByKeyInstruction', 6) |
| read = _messages.MessageField('ReadInstruction', 7) |
| systemName = _messages.StringField(8) |
| write = _messages.MessageField('WriteInstruction', 9) |
| |
| |
| class Parameter(_messages.Message): |
| r"""Structured data associated with this message. |
| |
| Fields: |
| key: Key or name for this parameter. |
| value: Value for this parameter. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| |
| class ParameterMetadata(_messages.Message): |
| r"""Metadata for a specific parameter. |
| |
| Fields: |
| helpText: Required. The help text to display for the parameter. |
| isOptional: Optional. Whether the parameter is optional. Defaults to |
| false. |
| label: Required. The label to display for the parameter. |
| name: Required. The name of the parameter. |
| regexes: Optional. Regexes that the parameter must match. |
| """ |
| |
| helpText = _messages.StringField(1) |
| isOptional = _messages.BooleanField(2) |
| label = _messages.StringField(3) |
| name = _messages.StringField(4) |
| regexes = _messages.StringField(5, repeated=True) |
| |
| |
| class PartialGroupByKeyInstruction(_messages.Message): |
| r"""An instruction that does a partial group-by-key. One input and one |
| output. |
| |
| Messages: |
| InputElementCodecValue: The codec to use for interpreting an element in |
| the input PTable. |
| ValueCombiningFnValue: The value combining function to invoke. |
| |
| Fields: |
| input: Describes the input to the partial group-by-key instruction. |
| inputElementCodec: The codec to use for interpreting an element in the |
| input PTable. |
| originalCombineValuesInputStoreName: If this instruction includes a |
| combining function this is the name of the intermediate store between |
| the GBK and the CombineValues. |
| originalCombineValuesStepName: If this instruction includes a combining |
| function, this is the name of the CombineValues instruction lifted into |
| this instruction. |
| sideInputs: Zero or more side inputs. |
| valueCombiningFn: The value combining function to invoke. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class InputElementCodecValue(_messages.Message): |
| r"""The codec to use for interpreting an element in the input PTable. |
| |
| Messages: |
| AdditionalProperty: An additional property for a InputElementCodecValue |
| object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a InputElementCodecValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class ValueCombiningFnValue(_messages.Message): |
| r"""The value combining function to invoke. |
| |
| Messages: |
| AdditionalProperty: An additional property for a ValueCombiningFnValue |
| object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a ValueCombiningFnValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| input = _messages.MessageField('InstructionInput', 1) |
| inputElementCodec = _messages.MessageField('InputElementCodecValue', 2) |
| originalCombineValuesInputStoreName = _messages.StringField(3) |
| originalCombineValuesStepName = _messages.StringField(4) |
| sideInputs = _messages.MessageField('SideInputInfo', 5, repeated=True) |
| valueCombiningFn = _messages.MessageField('ValueCombiningFnValue', 6) |
| |
| |
| class PipelineDescription(_messages.Message): |
| r"""A descriptive representation of submitted pipeline as well as the |
| executed form. This data is provided by the Dataflow service for ease of |
| visualizing the pipeline and interpreting Dataflow provided metrics. |
| |
| Fields: |
| displayData: Pipeline level display data. |
| executionPipelineStage: Description of each stage of execution of the |
| pipeline. |
| originalPipelineTransform: Description of each transform in the pipeline |
| and collections between them. |
| """ |
| |
| displayData = _messages.MessageField('DisplayData', 1, repeated=True) |
| executionPipelineStage = _messages.MessageField('ExecutionStageSummary', 2, repeated=True) |
| originalPipelineTransform = _messages.MessageField('TransformSummary', 3, repeated=True) |
| |
| |
| class Position(_messages.Message): |
| r"""Position defines a position within a collection of data. The value can |
| be either the end position, a key (used with ordered collections), a byte |
| offset, or a record index. |
| |
| Fields: |
| byteOffset: Position is a byte offset. |
| concatPosition: CloudPosition is a concat position. |
| end: Position is past all other positions. Also useful for the end |
| position of an unbounded range. |
| key: Position is a string key, ordered lexicographically. |
| recordIndex: Position is a record index. |
| shufflePosition: CloudPosition is a base64 encoded BatchShufflePosition |
| (with FIXED sharding). |
| """ |
| |
| byteOffset = _messages.IntegerField(1) |
| concatPosition = _messages.MessageField('ConcatPosition', 2) |
| end = _messages.BooleanField(3) |
| key = _messages.StringField(4) |
| recordIndex = _messages.IntegerField(5) |
| shufflePosition = _messages.StringField(6) |
| |
| |
| class PubSubIODetails(_messages.Message): |
| r"""Metadata for a PubSub connector used by the job. |
| |
| Fields: |
| subscription: Subscription used in the connection. |
| topic: Topic accessed in the connection. |
| """ |
| |
| subscription = _messages.StringField(1) |
| topic = _messages.StringField(2) |
| |
| |
| class PubsubLocation(_messages.Message): |
| r"""Identifies a pubsub location to use for transferring data into or out of |
| a streaming Dataflow job. |
| |
| Fields: |
| dropLateData: Indicates whether the pipeline allows late-arriving data. |
| idLabel: If set, contains a pubsub label from which to extract record ids. |
| If left empty, record deduplication will be strictly best effort. |
| subscription: A pubsub subscription, in the form of |
| "pubsub.googleapis.com/subscriptions/<project-id>/<subscription-name>" |
| timestampLabel: If set, contains a pubsub label from which to extract |
| record timestamps. If left empty, record timestamps will be generated |
| upon arrival. |
| topic: A pubsub topic, in the form of "pubsub.googleapis.com/topics |
| /<project-id>/<topic-name>" |
| trackingSubscription: If set, specifies the pubsub subscription that will |
| be used for tracking custom time timestamps for watermark estimation. |
| withAttributes: If true, then the client has requested to get pubsub |
| attributes. |
| """ |
| |
| dropLateData = _messages.BooleanField(1) |
| idLabel = _messages.StringField(2) |
| subscription = _messages.StringField(3) |
| timestampLabel = _messages.StringField(4) |
| topic = _messages.StringField(5) |
| trackingSubscription = _messages.StringField(6) |
| withAttributes = _messages.BooleanField(7) |
| |
| |
| class ReadInstruction(_messages.Message): |
| r"""An instruction that reads records. Takes no inputs, produces one output. |
| |
| Fields: |
| source: The source to read from. |
| """ |
| |
| source = _messages.MessageField('Source', 1) |
| |
| |
| class ReportWorkItemStatusRequest(_messages.Message): |
| r"""Request to report the status of WorkItems. |
| |
| Messages: |
| UnifiedWorkerRequestValue: Untranslated bag-of-bytes |
| WorkProgressUpdateRequest from UnifiedWorker. |
| |
| Fields: |
| currentWorkerTime: The current timestamp at the worker. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the WorkItem's job. |
| unifiedWorkerRequest: Untranslated bag-of-bytes WorkProgressUpdateRequest |
| from UnifiedWorker. |
| workItemStatuses: The order is unimportant, except that the order of the |
| WorkItemServiceState messages in the ReportWorkItemStatusResponse |
| corresponds to the order of WorkItemStatus messages here. |
| workerId: The ID of the worker reporting the WorkItem status. If this |
| does not match the ID of the worker which the Dataflow service believes |
| currently has the lease on the WorkItem, the report will be dropped |
| (with an error response). |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UnifiedWorkerRequestValue(_messages.Message): |
| r"""Untranslated bag-of-bytes WorkProgressUpdateRequest from |
| UnifiedWorker. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| UnifiedWorkerRequestValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. Contains field @type |
| with type URL. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UnifiedWorkerRequestValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| currentWorkerTime = _messages.StringField(1) |
| location = _messages.StringField(2) |
| unifiedWorkerRequest = _messages.MessageField('UnifiedWorkerRequestValue', 3) |
| workItemStatuses = _messages.MessageField('WorkItemStatus', 4, repeated=True) |
| workerId = _messages.StringField(5) |
| |
| |
| class ReportWorkItemStatusResponse(_messages.Message): |
| r"""Response from a request to report the status of WorkItems. |
| |
| Messages: |
| UnifiedWorkerResponseValue: Untranslated bag-of-bytes |
| WorkProgressUpdateResponse for UnifiedWorker. |
| |
| Fields: |
| unifiedWorkerResponse: Untranslated bag-of-bytes |
| WorkProgressUpdateResponse for UnifiedWorker. |
| workItemServiceStates: A set of messages indicating the service-side state |
| for each WorkItem whose status was reported, in the same order as the |
| WorkItemStatus messages in the ReportWorkItemStatusRequest which |
| resulting in this response. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UnifiedWorkerResponseValue(_messages.Message): |
| r"""Untranslated bag-of-bytes WorkProgressUpdateResponse for |
| UnifiedWorker. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| UnifiedWorkerResponseValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. Contains field @type |
| with type URL. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UnifiedWorkerResponseValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| unifiedWorkerResponse = _messages.MessageField('UnifiedWorkerResponseValue', 1) |
| workItemServiceStates = _messages.MessageField('WorkItemServiceState', 2, repeated=True) |
| |
| |
| class ReportedParallelism(_messages.Message): |
| r"""Represents the level of parallelism in a WorkItem's input, reported by |
| the worker. |
| |
| Fields: |
| isInfinite: Specifies whether the parallelism is infinite. If true, |
| "value" is ignored. Infinite parallelism means the service will assume |
| that the work item can always be split into more non-empty work items by |
| dynamic splitting. This is a work-around for lack of support for |
| infinity by the current JSON-based Java RPC stack. |
| value: Specifies the level of parallelism in case it is finite. |
| """ |
| |
| isInfinite = _messages.BooleanField(1) |
| value = _messages.FloatField(2) |
| |
| |
| class ResourceUtilizationReport(_messages.Message): |
| r"""Worker metrics exported from workers. This contains resource utilization |
| metrics accumulated from a variety of sources. For more information, see go |
| /df-resource-signals. |
| |
| Fields: |
| cpuTime: CPU utilization samples. |
| """ |
| |
| cpuTime = _messages.MessageField('CPUTime', 1, repeated=True) |
| |
| |
| class ResourceUtilizationReportResponse(_messages.Message): |
| r"""Service-side response to WorkerMessage reporting resource utilization. |
| """ |
| |
| |
| |
| class RuntimeEnvironment(_messages.Message): |
| r"""The environment values to set at runtime. |
| |
| Messages: |
| AdditionalUserLabelsValue: Additional user labels to be specified for the |
| job. Keys and values should follow the restrictions specified in the |
| [labeling restrictions](https://cloud.google.com/compute/docs/labeling- |
| resources#restrictions) page. |
| |
| Fields: |
| additionalExperiments: Additional experiment flags for the job. |
| additionalUserLabels: Additional user labels to be specified for the job. |
| Keys and values should follow the restrictions specified in the |
| [labeling restrictions](https://cloud.google.com/compute/docs/labeling- |
| resources#restrictions) page. |
| bypassTempDirValidation: Whether to bypass the safety checks for the job's |
| temporary directory. Use with caution. |
| machineType: The machine type to use for the job. Defaults to the value |
| from the template if not specified. |
| maxWorkers: The maximum number of Google Compute Engine instances to be |
| made available to your pipeline during execution, from 1 to 1000. |
| network: Network to which VMs will be assigned. If empty or unspecified, |
| the service will use the network "default". |
| numWorkers: The initial number of Google Compute Engine instnaces for the |
| job. |
| serviceAccountEmail: The email address of the service account to run the |
| job as. |
| subnetwork: Subnetwork to which VMs will be assigned, if desired. |
| Expected to be of the form "regions/REGION/subnetworks/SUBNETWORK". |
| tempLocation: The Cloud Storage path to use for temporary files. Must be a |
| valid Cloud Storage URL, beginning with `gs://`. |
| zone: The Compute Engine [availability |
| zone](https://cloud.google.com/compute/docs/regions-zones/regions-zones) |
| for launching worker instances to run your pipeline. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class AdditionalUserLabelsValue(_messages.Message): |
| r"""Additional user labels to be specified for the job. Keys and values |
| should follow the restrictions specified in the [labeling |
| restrictions](https://cloud.google.com/compute/docs/labeling- |
| resources#restrictions) page. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| AdditionalUserLabelsValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type |
| AdditionalUserLabelsValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a AdditionalUserLabelsValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| additionalExperiments = _messages.StringField(1, repeated=True) |
| additionalUserLabels = _messages.MessageField('AdditionalUserLabelsValue', 2) |
| bypassTempDirValidation = _messages.BooleanField(3) |
| machineType = _messages.StringField(4) |
| maxWorkers = _messages.IntegerField(5, variant=_messages.Variant.INT32) |
| network = _messages.StringField(6) |
| numWorkers = _messages.IntegerField(7, variant=_messages.Variant.INT32) |
| serviceAccountEmail = _messages.StringField(8) |
| subnetwork = _messages.StringField(9) |
| tempLocation = _messages.StringField(10) |
| zone = _messages.StringField(11) |
| |
| |
| class SdkVersion(_messages.Message): |
| r"""The version of the SDK used to run the job. |
| |
| Enums: |
| SdkSupportStatusValueValuesEnum: The support status for this SDK version. |
| |
| Fields: |
| sdkSupportStatus: The support status for this SDK version. |
| version: The version of the SDK used to run the job. |
| versionDisplayName: A readable string describing the version of the SDK. |
| """ |
| |
| class SdkSupportStatusValueValuesEnum(_messages.Enum): |
| r"""The support status for this SDK version. |
| |
| Values: |
| UNKNOWN: Cloud Dataflow is unaware of this version. |
| SUPPORTED: This is a known version of an SDK, and is supported. |
| STALE: A newer version of the SDK family exists, and an update is |
| recommended. |
| DEPRECATED: This version of the SDK is deprecated and will eventually be |
| no longer supported. |
| UNSUPPORTED: Support for this SDK version has ended and it should no |
| longer be used. |
| """ |
| UNKNOWN = 0 |
| SUPPORTED = 1 |
| STALE = 2 |
| DEPRECATED = 3 |
| UNSUPPORTED = 4 |
| |
| sdkSupportStatus = _messages.EnumField('SdkSupportStatusValueValuesEnum', 1) |
| version = _messages.StringField(2) |
| versionDisplayName = _messages.StringField(3) |
| |
| |
| class SendDebugCaptureRequest(_messages.Message): |
| r"""Request to send encoded debug information. |
| |
| Fields: |
| componentId: The internal component id for which debug information is |
| sent. |
| data: The encoded debug information. |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job specified by job_id. |
| workerId: The worker id, i.e., VM hostname. |
| """ |
| |
| componentId = _messages.StringField(1) |
| data = _messages.StringField(2) |
| location = _messages.StringField(3) |
| workerId = _messages.StringField(4) |
| |
| |
| class SendDebugCaptureResponse(_messages.Message): |
| r"""Response to a send capture request. |
| nothing""" |
| |
| |
| class SendWorkerMessagesRequest(_messages.Message): |
| r"""A request for sending worker messages to the service. |
| |
| Fields: |
| location: The [regional endpoint] |
| (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) |
| that contains the job. |
| workerMessages: The WorkerMessages to send. |
| """ |
| |
| location = _messages.StringField(1) |
| workerMessages = _messages.MessageField('WorkerMessage', 2, repeated=True) |
| |
| |
| class SendWorkerMessagesResponse(_messages.Message): |
| r"""The response to the worker messages. |
| |
| Fields: |
| workerMessageResponses: The servers response to the worker messages. |
| """ |
| |
| workerMessageResponses = _messages.MessageField('WorkerMessageResponse', 1, repeated=True) |
| |
| |
| class SeqMapTask(_messages.Message): |
| r"""Describes a particular function to invoke. |
| |
| Messages: |
| UserFnValue: The user function to invoke. |
| |
| Fields: |
| inputs: Information about each of the inputs. |
| name: The user-provided name of the SeqDo operation. |
| outputInfos: Information about each of the outputs. |
| stageName: System-defined name of the stage containing the SeqDo |
| operation. Unique across the workflow. |
| systemName: System-defined name of the SeqDo operation. Unique across the |
| workflow. |
| userFn: The user function to invoke. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UserFnValue(_messages.Message): |
| r"""The user function to invoke. |
| |
| Messages: |
| AdditionalProperty: An additional property for a UserFnValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UserFnValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| inputs = _messages.MessageField('SideInputInfo', 1, repeated=True) |
| name = _messages.StringField(2) |
| outputInfos = _messages.MessageField('SeqMapTaskOutputInfo', 3, repeated=True) |
| stageName = _messages.StringField(4) |
| systemName = _messages.StringField(5) |
| userFn = _messages.MessageField('UserFnValue', 6) |
| |
| |
| class SeqMapTaskOutputInfo(_messages.Message): |
| r"""Information about an output of a SeqMapTask. |
| |
| Fields: |
| sink: The sink to write the output value to. |
| tag: The id of the TupleTag the user code will tag the output value by. |
| """ |
| |
| sink = _messages.MessageField('Sink', 1) |
| tag = _messages.StringField(2) |
| |
| |
| class ShellTask(_messages.Message): |
| r"""A task which consists of a shell command for the worker to execute. |
| |
| Fields: |
| command: The shell command to run. |
| exitCode: Exit code for the task. |
| """ |
| |
| command = _messages.StringField(1) |
| exitCode = _messages.IntegerField(2, variant=_messages.Variant.INT32) |
| |
| |
| class SideInputInfo(_messages.Message): |
| r"""Information about a side input of a DoFn or an input of a SeqDoFn. |
| |
| Messages: |
| KindValue: How to interpret the source element(s) as a side input value. |
| |
| Fields: |
| kind: How to interpret the source element(s) as a side input value. |
| sources: The source(s) to read element(s) from to get the value of this |
| side input. If more than one source, then the elements are taken from |
| the sources, in the specified order if order matters. At least one |
| source is required. |
| tag: The id of the tag the user code will access this side input by; this |
| should correspond to the tag of some MultiOutputInfo. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class KindValue(_messages.Message): |
| r"""How to interpret the source element(s) as a side input value. |
| |
| Messages: |
| AdditionalProperty: An additional property for a KindValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a KindValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| kind = _messages.MessageField('KindValue', 1) |
| sources = _messages.MessageField('Source', 2, repeated=True) |
| tag = _messages.StringField(3) |
| |
| |
| class Sink(_messages.Message): |
| r"""A sink that records can be encoded and written to. |
| |
| Messages: |
| CodecValue: The codec to use to encode data written to the sink. |
| SpecValue: The sink to write to, plus its parameters. |
| |
| Fields: |
| codec: The codec to use to encode data written to the sink. |
| spec: The sink to write to, plus its parameters. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class CodecValue(_messages.Message): |
| r"""The codec to use to encode data written to the sink. |
| |
| Messages: |
| AdditionalProperty: An additional property for a CodecValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a CodecValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class SpecValue(_messages.Message): |
| r"""The sink to write to, plus its parameters. |
| |
| Messages: |
| AdditionalProperty: An additional property for a SpecValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a SpecValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| codec = _messages.MessageField('CodecValue', 1) |
| spec = _messages.MessageField('SpecValue', 2) |
| |
| |
| class Snapshot(_messages.Message): |
| r"""Represents a snapshot of a job. |
| |
| Enums: |
| StateValueValuesEnum: State of the snapshot. |
| |
| Fields: |
| creationTime: The time this snapshot was created. |
| id: The unique ID of this snapshot. |
| projectId: The project this snapshot belongs to. |
| sourceJobId: The job this snapshot was created from. |
| state: State of the snapshot. |
| ttl: The time after which this snapshot will be automatically deleted. |
| """ |
| |
| class StateValueValuesEnum(_messages.Enum): |
| r"""State of the snapshot. |
| |
| Values: |
| UNKNOWN_SNAPSHOT_STATE: Unknown state. |
| PENDING: Snapshot intent to create has been persisted, snapshotting of |
| state has not yet started. |
| RUNNING: Snapshotting is being performed. |
| READY: Snapshot has been created and is ready to be used. |
| FAILED: Snapshot failed to be created. |
| DELETED: Snapshot has been deleted. |
| """ |
| UNKNOWN_SNAPSHOT_STATE = 0 |
| PENDING = 1 |
| RUNNING = 2 |
| READY = 3 |
| FAILED = 4 |
| DELETED = 5 |
| |
| creationTime = _messages.StringField(1) |
| id = _messages.StringField(2) |
| projectId = _messages.StringField(3) |
| sourceJobId = _messages.StringField(4) |
| state = _messages.EnumField('StateValueValuesEnum', 5) |
| ttl = _messages.StringField(6) |
| |
| |
| class SnapshotJobRequest(_messages.Message): |
| r"""Request to create a snapshot of a job. |
| |
| Fields: |
| location: The location that contains this job. |
| ttl: TTL for the snapshot. |
| """ |
| |
| location = _messages.StringField(1) |
| ttl = _messages.StringField(2) |
| |
| |
| class Source(_messages.Message): |
| r"""A source that records can be read and decoded from. |
| |
| Messages: |
| BaseSpecsValueListEntry: A BaseSpecsValueListEntry object. |
| CodecValue: The codec to use to decode data read from the source. |
| SpecValue: The source to read from, plus its parameters. |
| |
| Fields: |
| baseSpecs: While splitting, sources may specify the produced bundles as |
| differences against another source, in order to save backend-side memory |
| and allow bigger jobs. For details, see SourceSplitRequest. To support |
| this use case, the full set of parameters of the source is logically |
| obtained by taking the latest explicitly specified value of each |
| parameter in the order: base_specs (later items win), spec (overrides |
| anything in base_specs). |
| codec: The codec to use to decode data read from the source. |
| doesNotNeedSplitting: Setting this value to true hints to the framework |
| that the source doesn't need splitting, and using SourceSplitRequest on |
| it would yield SOURCE_SPLIT_OUTCOME_USE_CURRENT. E.g. a file splitter |
| may set this to true when splitting a single file into a set of byte |
| ranges of appropriate size, and set this to false when splitting a |
| filepattern into individual files. However, for efficiency, a file |
| splitter may decide to produce file subranges directly from the |
| filepattern to avoid a splitting round-trip. See SourceSplitRequest for |
| an overview of the splitting process. This field is meaningful only in |
| the Source objects populated by the user (e.g. when filling in a |
| DerivedSource). Source objects supplied by the framework to the user |
| don't have this field populated. |
| metadata: Optionally, metadata for this source can be supplied right away, |
| avoiding a SourceGetMetadataOperation roundtrip (see |
| SourceOperationRequest). This field is meaningful only in the Source |
| objects populated by the user (e.g. when filling in a DerivedSource). |
| Source objects supplied by the framework to the user don't have this |
| field populated. |
| spec: The source to read from, plus its parameters. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class BaseSpecsValueListEntry(_messages.Message): |
| r"""A BaseSpecsValueListEntry object. |
| |
| Messages: |
| AdditionalProperty: An additional property for a BaseSpecsValueListEntry |
| object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a BaseSpecsValueListEntry object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class CodecValue(_messages.Message): |
| r"""The codec to use to decode data read from the source. |
| |
| Messages: |
| AdditionalProperty: An additional property for a CodecValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a CodecValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class SpecValue(_messages.Message): |
| r"""The source to read from, plus its parameters. |
| |
| Messages: |
| AdditionalProperty: An additional property for a SpecValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a SpecValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| baseSpecs = _messages.MessageField('BaseSpecsValueListEntry', 1, repeated=True) |
| codec = _messages.MessageField('CodecValue', 2) |
| doesNotNeedSplitting = _messages.BooleanField(3) |
| metadata = _messages.MessageField('SourceMetadata', 4) |
| spec = _messages.MessageField('SpecValue', 5) |
| |
| |
| class SourceFork(_messages.Message): |
| r"""DEPRECATED in favor of DynamicSourceSplit. |
| |
| Fields: |
| primary: DEPRECATED |
| primarySource: DEPRECATED |
| residual: DEPRECATED |
| residualSource: DEPRECATED |
| """ |
| |
| primary = _messages.MessageField('SourceSplitShard', 1) |
| primarySource = _messages.MessageField('DerivedSource', 2) |
| residual = _messages.MessageField('SourceSplitShard', 3) |
| residualSource = _messages.MessageField('DerivedSource', 4) |
| |
| |
| class SourceGetMetadataRequest(_messages.Message): |
| r"""A request to compute the SourceMetadata of a Source. |
| |
| Fields: |
| source: Specification of the source whose metadata should be computed. |
| """ |
| |
| source = _messages.MessageField('Source', 1) |
| |
| |
| class SourceGetMetadataResponse(_messages.Message): |
| r"""The result of a SourceGetMetadataOperation. |
| |
| Fields: |
| metadata: The computed metadata. |
| """ |
| |
| metadata = _messages.MessageField('SourceMetadata', 1) |
| |
| |
| class SourceMetadata(_messages.Message): |
| r"""Metadata about a Source useful for automatically optimizing and tuning |
| the pipeline, etc. |
| |
| Fields: |
| estimatedSizeBytes: An estimate of the total size (in bytes) of the data |
| that would be read from this source. This estimate is in terms of |
| external storage size, before any decompression or other processing done |
| by the reader. |
| infinite: Specifies that the size of this source is known to be infinite |
| (this is a streaming source). |
| producesSortedKeys: Whether this source is known to produce key/value |
| pairs with the (encoded) keys in lexicographically sorted order. |
| """ |
| |
| estimatedSizeBytes = _messages.IntegerField(1) |
| infinite = _messages.BooleanField(2) |
| producesSortedKeys = _messages.BooleanField(3) |
| |
| |
| class SourceOperationRequest(_messages.Message): |
| r"""A work item that represents the different operations that can be |
| performed on a user-defined Source specification. |
| |
| Fields: |
| getMetadata: Information about a request to get metadata about a source. |
| name: User-provided name of the Read instruction for this source. |
| originalName: System-defined name for the Read instruction for this source |
| in the original workflow graph. |
| split: Information about a request to split a source. |
| stageName: System-defined name of the stage containing the source |
| operation. Unique across the workflow. |
| systemName: System-defined name of the Read instruction for this source. |
| Unique across the workflow. |
| """ |
| |
| getMetadata = _messages.MessageField('SourceGetMetadataRequest', 1) |
| name = _messages.StringField(2) |
| originalName = _messages.StringField(3) |
| split = _messages.MessageField('SourceSplitRequest', 4) |
| stageName = _messages.StringField(5) |
| systemName = _messages.StringField(6) |
| |
| |
| class SourceOperationResponse(_messages.Message): |
| r"""The result of a SourceOperationRequest, specified in |
| ReportWorkItemStatusRequest.source_operation when the work item is |
| completed. |
| |
| Fields: |
| getMetadata: A response to a request to get metadata about a source. |
| split: A response to a request to split a source. |
| """ |
| |
| getMetadata = _messages.MessageField('SourceGetMetadataResponse', 1) |
| split = _messages.MessageField('SourceSplitResponse', 2) |
| |
| |
| class SourceSplitOptions(_messages.Message): |
| r"""Hints for splitting a Source into bundles (parts for parallel |
| processing) using SourceSplitRequest. |
| |
| Fields: |
| desiredBundleSizeBytes: The source should be split into a set of bundles |
| where the estimated size of each is approximately this many bytes. |
| desiredShardSizeBytes: DEPRECATED in favor of desired_bundle_size_bytes. |
| """ |
| |
| desiredBundleSizeBytes = _messages.IntegerField(1) |
| desiredShardSizeBytes = _messages.IntegerField(2) |
| |
| |
| class SourceSplitRequest(_messages.Message): |
| r"""Represents the operation to split a high-level Source specification into |
| bundles (parts for parallel processing). At a high level, splitting of a |
| source into bundles happens as follows: SourceSplitRequest is applied to the |
| source. If it returns SOURCE_SPLIT_OUTCOME_USE_CURRENT, no further splitting |
| happens and the source is used "as is". Otherwise, splitting is applied |
| recursively to each produced DerivedSource. As an optimization, for any |
| Source, if its does_not_need_splitting is true, the framework assumes that |
| splitting this source would return SOURCE_SPLIT_OUTCOME_USE_CURRENT, and |
| doesn't initiate a SourceSplitRequest. This applies both to the initial |
| source being split and to bundles produced from it. |
| |
| Fields: |
| options: Hints for tuning the splitting process. |
| source: Specification of the source to be split. |
| """ |
| |
| options = _messages.MessageField('SourceSplitOptions', 1) |
| source = _messages.MessageField('Source', 2) |
| |
| |
| class SourceSplitResponse(_messages.Message): |
| r"""The response to a SourceSplitRequest. |
| |
| Enums: |
| OutcomeValueValuesEnum: Indicates whether splitting happened and produced |
| a list of bundles. If this is USE_CURRENT_SOURCE_AS_IS, the current |
| source should be processed "as is" without splitting. "bundles" is |
| ignored in this case. If this is SPLITTING_HAPPENED, then "bundles" |
| contains a list of bundles into which the source was split. |
| |
| Fields: |
| bundles: If outcome is SPLITTING_HAPPENED, then this is a list of bundles |
| into which the source was split. Otherwise this field is ignored. This |
| list can be empty, which means the source represents an empty input. |
| outcome: Indicates whether splitting happened and produced a list of |
| bundles. If this is USE_CURRENT_SOURCE_AS_IS, the current source should |
| be processed "as is" without splitting. "bundles" is ignored in this |
| case. If this is SPLITTING_HAPPENED, then "bundles" contains a list of |
| bundles into which the source was split. |
| shards: DEPRECATED in favor of bundles. |
| """ |
| |
| class OutcomeValueValuesEnum(_messages.Enum): |
| r"""Indicates whether splitting happened and produced a list of bundles. |
| If this is USE_CURRENT_SOURCE_AS_IS, the current source should be |
| processed "as is" without splitting. "bundles" is ignored in this case. If |
| this is SPLITTING_HAPPENED, then "bundles" contains a list of bundles into |
| which the source was split. |
| |
| Values: |
| SOURCE_SPLIT_OUTCOME_UNKNOWN: The source split outcome is unknown, or |
| unspecified. |
| SOURCE_SPLIT_OUTCOME_USE_CURRENT: The current source should be processed |
| "as is" without splitting. |
| SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED: Splitting produced a list of |
| bundles. |
| """ |
| SOURCE_SPLIT_OUTCOME_UNKNOWN = 0 |
| SOURCE_SPLIT_OUTCOME_USE_CURRENT = 1 |
| SOURCE_SPLIT_OUTCOME_SPLITTING_HAPPENED = 2 |
| |
| bundles = _messages.MessageField('DerivedSource', 1, repeated=True) |
| outcome = _messages.EnumField('OutcomeValueValuesEnum', 2) |
| shards = _messages.MessageField('SourceSplitShard', 3, repeated=True) |
| |
| |
| class SourceSplitShard(_messages.Message): |
| r"""DEPRECATED in favor of DerivedSource. |
| |
| Enums: |
| DerivationModeValueValuesEnum: DEPRECATED |
| |
| Fields: |
| derivationMode: DEPRECATED |
| source: DEPRECATED |
| """ |
| |
| class DerivationModeValueValuesEnum(_messages.Enum): |
| r"""DEPRECATED |
| |
| Values: |
| SOURCE_DERIVATION_MODE_UNKNOWN: The source derivation is unknown, or |
| unspecified. |
| SOURCE_DERIVATION_MODE_INDEPENDENT: Produce a completely independent |
| Source with no base. |
| SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT: Produce a Source based on the |
| Source being split. |
| SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT: Produce a Source based on the |
| base of the Source being split. |
| """ |
| SOURCE_DERIVATION_MODE_UNKNOWN = 0 |
| SOURCE_DERIVATION_MODE_INDEPENDENT = 1 |
| SOURCE_DERIVATION_MODE_CHILD_OF_CURRENT = 2 |
| SOURCE_DERIVATION_MODE_SIBLING_OF_CURRENT = 3 |
| |
| derivationMode = _messages.EnumField('DerivationModeValueValuesEnum', 1) |
| source = _messages.MessageField('Source', 2) |
| |
| |
| class SpannerIODetails(_messages.Message): |
| r"""Metadata for a Spanner connector used by the job. |
| |
| Fields: |
| databaseId: DatabaseId accessed in the connection. |
| instanceId: InstanceId accessed in the connection. |
| projectId: ProjectId accessed in the connection. |
| """ |
| |
| databaseId = _messages.StringField(1) |
| instanceId = _messages.StringField(2) |
| projectId = _messages.StringField(3) |
| |
| |
| class SplitInt64(_messages.Message): |
| r"""A representation of an int64, n, that is immune to precision loss when |
| encoded in JSON. |
| |
| Fields: |
| highBits: The high order bits, including the sign: n >> 32. |
| lowBits: The low order bits: n & 0xffffffff. |
| """ |
| |
| highBits = _messages.IntegerField(1, variant=_messages.Variant.INT32) |
| lowBits = _messages.IntegerField(2, variant=_messages.Variant.UINT32) |
| |
| |
| class StageSource(_messages.Message): |
| r"""Description of an input or output of an execution stage. |
| |
| Fields: |
| name: Dataflow service generated name for this source. |
| originalTransformOrCollection: User name for the original user transform |
| or collection with which this source is most closely associated. |
| sizeBytes: Size of the source, if measurable. |
| userName: Human-readable name for this source; may be user or system |
| generated. |
| """ |
| |
| name = _messages.StringField(1) |
| originalTransformOrCollection = _messages.StringField(2) |
| sizeBytes = _messages.IntegerField(3) |
| userName = _messages.StringField(4) |
| |
| |
| class StandardQueryParameters(_messages.Message): |
| r"""Query parameters accepted by all methods. |
| |
| Enums: |
| FXgafvValueValuesEnum: V1 error format. |
| AltValueValuesEnum: Data format for response. |
| |
| Fields: |
| f__xgafv: V1 error format. |
| access_token: OAuth access token. |
| alt: Data format for response. |
| callback: JSONP |
| fields: Selector specifying which fields to include in a partial response. |
| key: API key. Your API key identifies your project and provides you with |
| API access, quota, and reports. Required unless you provide an OAuth 2.0 |
| token. |
| oauth_token: OAuth 2.0 token for the current user. |
| prettyPrint: Returns response with indentations and line breaks. |
| quotaUser: Available to use for quota purposes for server-side |
| applications. Can be any arbitrary string assigned to a user, but should |
| not exceed 40 characters. |
| trace: A tracing token of the form "token:<tokenid>" to include in api |
| requests. |
| uploadType: Legacy upload protocol for media (e.g. "media", "multipart"). |
| upload_protocol: Upload protocol for media (e.g. "raw", "multipart"). |
| """ |
| |
| class AltValueValuesEnum(_messages.Enum): |
| r"""Data format for response. |
| |
| Values: |
| json: Responses with Content-Type of application/json |
| media: Media download with context-dependent Content-Type |
| proto: Responses with Content-Type of application/x-protobuf |
| """ |
| json = 0 |
| media = 1 |
| proto = 2 |
| |
| class FXgafvValueValuesEnum(_messages.Enum): |
| r"""V1 error format. |
| |
| Values: |
| _1: v1 error format |
| _2: v2 error format |
| """ |
| _1 = 0 |
| _2 = 1 |
| |
| f__xgafv = _messages.EnumField('FXgafvValueValuesEnum', 1) |
| access_token = _messages.StringField(2) |
| alt = _messages.EnumField('AltValueValuesEnum', 3, default=u'json') |
| callback = _messages.StringField(4) |
| fields = _messages.StringField(5) |
| key = _messages.StringField(6) |
| oauth_token = _messages.StringField(7) |
| prettyPrint = _messages.BooleanField(8, default=True) |
| quotaUser = _messages.StringField(9) |
| trace = _messages.StringField(10) |
| uploadType = _messages.StringField(11) |
| upload_protocol = _messages.StringField(12) |
| |
| |
| class StateFamilyConfig(_messages.Message): |
| r"""State family configuration. |
| |
| Fields: |
| isRead: If true, this family corresponds to a read operation. |
| stateFamily: The state family value. |
| """ |
| |
| isRead = _messages.BooleanField(1) |
| stateFamily = _messages.StringField(2) |
| |
| |
| class Status(_messages.Message): |
| r"""The `Status` type defines a logical error model that is suitable for |
| different programming environments, including REST APIs and RPC APIs. It is |
| used by [gRPC](https://github.com/grpc). The error model is designed to be: |
| - Simple to use and understand for most users - Flexible enough to meet |
| unexpected needs # Overview The `Status` message contains three pieces of |
| data: error code, error message, and error details. The error code should be |
| an enum value of google.rpc.Code, but it may accept additional error codes |
| if needed. The error message should be a developer-facing English message |
| that helps developers *understand* and *resolve* the error. If a localized |
| user-facing error message is needed, put the localized message in the error |
| details or localize it in the client. The optional error details may contain |
| arbitrary information about the error. There is a predefined set of error |
| detail types in the package `google.rpc` that can be used for common error |
| conditions. # Language mapping The `Status` message is the logical |
| representation of the error model, but it is not necessarily the actual wire |
| format. When the `Status` message is exposed in different client libraries |
| and different wire protocols, it can be mapped differently. For example, it |
| will likely be mapped to some exceptions in Java, but more likely mapped to |
| some error codes in C. # Other uses The error model and the `Status` |
| message can be used in a variety of environments, either with or without |
| APIs, to provide a consistent developer experience across different |
| environments. Example uses of this error model include: - Partial errors. |
| If a service needs to return partial errors to the client, it may embed |
| the `Status` in the normal response to indicate the partial errors. - |
| Workflow errors. A typical workflow has multiple steps. Each step may |
| have a `Status` message for error reporting. - Batch operations. If a |
| client uses batch request and batch response, the `Status` message |
| should be used directly inside batch response, one for each error sub- |
| response. - Asynchronous operations. If an API call embeds asynchronous |
| operation results in its response, the status of those operations should |
| be represented directly using the `Status` message. - Logging. If some |
| API errors are stored in logs, the message `Status` could be used |
| directly after any stripping needed for security/privacy reasons. |
| |
| Messages: |
| DetailsValueListEntry: A DetailsValueListEntry object. |
| |
| Fields: |
| code: The status code, which should be an enum value of google.rpc.Code. |
| details: A list of messages that carry the error details. There is a |
| common set of message types for APIs to use. |
| message: A developer-facing error message, which should be in English. Any |
| user-facing error message should be localized and sent in the |
| google.rpc.Status.details field, or localized by the client. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class DetailsValueListEntry(_messages.Message): |
| r"""A DetailsValueListEntry object. |
| |
| Messages: |
| AdditionalProperty: An additional property for a DetailsValueListEntry |
| object. |
| |
| Fields: |
| additionalProperties: Properties of the object. Contains field @type |
| with type URL. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a DetailsValueListEntry object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| code = _messages.IntegerField(1, variant=_messages.Variant.INT32) |
| details = _messages.MessageField('DetailsValueListEntry', 2, repeated=True) |
| message = _messages.StringField(3) |
| |
| |
| class Step(_messages.Message): |
| r"""Defines a particular step within a Cloud Dataflow job. A job consists |
| of multiple steps, each of which performs some specific operation as part of |
| the overall job. Data is typically passed from one step to another as part |
| of the job. Here's an example of a sequence of steps which together |
| implement a Map-Reduce job: * Read a collection of data from some source, |
| parsing the collection's elements. * Validate the elements. * |
| Apply a user-defined function to map each element to some value and |
| extract an element-specific key value. * Group elements with the same key |
| into a single element with that key, transforming a multiply-keyed |
| collection into a uniquely-keyed collection. * Write the elements out |
| to some data sink. Note that the Cloud Dataflow service may be used to run |
| many different types of jobs, not just Map-Reduce. |
| |
| Messages: |
| PropertiesValue: Named properties associated with the step. Each kind of |
| predefined step has its own required set of properties. Must be provided |
| on Create. Only retrieved with JOB_VIEW_ALL. |
| |
| Fields: |
| kind: The kind of step in the Cloud Dataflow job. |
| name: The name that identifies the step. This must be unique for each step |
| with respect to all other steps in the Cloud Dataflow job. |
| properties: Named properties associated with the step. Each kind of |
| predefined step has its own required set of properties. Must be provided |
| on Create. Only retrieved with JOB_VIEW_ALL. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class PropertiesValue(_messages.Message): |
| r"""Named properties associated with the step. Each kind of predefined |
| step has its own required set of properties. Must be provided on Create. |
| Only retrieved with JOB_VIEW_ALL. |
| |
| Messages: |
| AdditionalProperty: An additional property for a PropertiesValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a PropertiesValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| kind = _messages.StringField(1) |
| name = _messages.StringField(2) |
| properties = _messages.MessageField('PropertiesValue', 3) |
| |
| |
| class StreamLocation(_messages.Message): |
| r"""Describes a stream of data, either as input to be processed or as output |
| of a streaming Dataflow job. |
| |
| Fields: |
| customSourceLocation: The stream is a custom source. |
| pubsubLocation: The stream is a pubsub stream. |
| sideInputLocation: The stream is a streaming side input. |
| streamingStageLocation: The stream is part of another computation within |
| the current streaming Dataflow job. |
| """ |
| |
| customSourceLocation = _messages.MessageField('CustomSourceLocation', 1) |
| pubsubLocation = _messages.MessageField('PubsubLocation', 2) |
| sideInputLocation = _messages.MessageField('StreamingSideInputLocation', 3) |
| streamingStageLocation = _messages.MessageField('StreamingStageLocation', 4) |
| |
| |
| class StreamingApplianceSnapshotConfig(_messages.Message): |
| r"""Streaming appliance snapshot configuration. |
| |
| Fields: |
| importStateEndpoint: Indicates which endpoint is used to import appliance |
| state. |
| snapshotId: If set, indicates the snapshot id for the snapshot being |
| performed. |
| """ |
| |
| importStateEndpoint = _messages.StringField(1) |
| snapshotId = _messages.StringField(2) |
| |
| |
| class StreamingComputationConfig(_messages.Message): |
| r"""Configuration information for a single streaming computation. |
| |
| Messages: |
| TransformUserNameToStateFamilyValue: Map from user name of stateful |
| transforms in this stage to their state family. |
| |
| Fields: |
| computationId: Unique identifier for this computation. |
| instructions: Instructions that comprise the computation. |
| stageName: Stage name of this computation. |
| systemName: System defined name for this computation. |
| transformUserNameToStateFamily: Map from user name of stateful transforms |
| in this stage to their state family. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class TransformUserNameToStateFamilyValue(_messages.Message): |
| r"""Map from user name of stateful transforms in this stage to their state |
| family. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| TransformUserNameToStateFamilyValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type |
| TransformUserNameToStateFamilyValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a TransformUserNameToStateFamilyValue |
| object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| computationId = _messages.StringField(1) |
| instructions = _messages.MessageField('ParallelInstruction', 2, repeated=True) |
| stageName = _messages.StringField(3) |
| systemName = _messages.StringField(4) |
| transformUserNameToStateFamily = _messages.MessageField('TransformUserNameToStateFamilyValue', 5) |
| |
| |
| class StreamingComputationRanges(_messages.Message): |
| r"""Describes full or partial data disk assignment information of the |
| computation ranges. |
| |
| Fields: |
| computationId: The ID of the computation. |
| rangeAssignments: Data disk assignments for ranges from this computation. |
| """ |
| |
| computationId = _messages.StringField(1) |
| rangeAssignments = _messages.MessageField('KeyRangeDataDiskAssignment', 2, repeated=True) |
| |
| |
| class StreamingComputationTask(_messages.Message): |
| r"""A task which describes what action should be performed for the specified |
| streaming computation ranges. |
| |
| Enums: |
| TaskTypeValueValuesEnum: A type of streaming computation task. |
| |
| Fields: |
| computationRanges: Contains ranges of a streaming computation this task |
| should apply to. |
| dataDisks: Describes the set of data disks this task should apply to. |
| taskType: A type of streaming computation task. |
| """ |
| |
| class TaskTypeValueValuesEnum(_messages.Enum): |
| r"""A type of streaming computation task. |
| |
| Values: |
| STREAMING_COMPUTATION_TASK_UNKNOWN: The streaming computation task is |
| unknown, or unspecified. |
| STREAMING_COMPUTATION_TASK_STOP: Stop processing specified streaming |
| computation range(s). |
| STREAMING_COMPUTATION_TASK_START: Start processing specified streaming |
| computation range(s). |
| """ |
| STREAMING_COMPUTATION_TASK_UNKNOWN = 0 |
| STREAMING_COMPUTATION_TASK_STOP = 1 |
| STREAMING_COMPUTATION_TASK_START = 2 |
| |
| computationRanges = _messages.MessageField('StreamingComputationRanges', 1, repeated=True) |
| dataDisks = _messages.MessageField('MountedDataDisk', 2, repeated=True) |
| taskType = _messages.EnumField('TaskTypeValueValuesEnum', 3) |
| |
| |
| class StreamingConfigTask(_messages.Message): |
| r"""A task that carries configuration information for streaming |
| computations. |
| |
| Messages: |
| UserStepToStateFamilyNameMapValue: Map from user step names to state |
| families. |
| |
| Fields: |
| maxWorkItemCommitBytes: Maximum size for work item commit supported |
| windmill storage layer. |
| streamingComputationConfigs: Set of computation configuration information. |
| userStepToStateFamilyNameMap: Map from user step names to state families. |
| windmillServiceEndpoint: If present, the worker must use this endpoint to |
| communicate with Windmill Service dispatchers, otherwise the worker must |
| continue to use whatever endpoint it had been using. |
| windmillServicePort: If present, the worker must use this port to |
| communicate with Windmill Service dispatchers. Only applicable when |
| windmill_service_endpoint is specified. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UserStepToStateFamilyNameMapValue(_messages.Message): |
| r"""Map from user step names to state families. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| UserStepToStateFamilyNameMapValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type |
| UserStepToStateFamilyNameMapValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UserStepToStateFamilyNameMapValue |
| object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| maxWorkItemCommitBytes = _messages.IntegerField(1) |
| streamingComputationConfigs = _messages.MessageField('StreamingComputationConfig', 2, repeated=True) |
| userStepToStateFamilyNameMap = _messages.MessageField('UserStepToStateFamilyNameMapValue', 3) |
| windmillServiceEndpoint = _messages.StringField(4) |
| windmillServicePort = _messages.IntegerField(5) |
| |
| |
| class StreamingSetupTask(_messages.Message): |
| r"""A task which initializes part of a streaming Dataflow job. |
| |
| Fields: |
| drain: The user has requested drain. |
| receiveWorkPort: The TCP port on which the worker should listen for |
| messages from other streaming computation workers. |
| snapshotConfig: Configures streaming appliance snapshot. |
| streamingComputationTopology: The global topology of the streaming |
| Dataflow job. |
| workerHarnessPort: The TCP port used by the worker to communicate with the |
| Dataflow worker harness. |
| """ |
| |
| drain = _messages.BooleanField(1) |
| receiveWorkPort = _messages.IntegerField(2, variant=_messages.Variant.INT32) |
| snapshotConfig = _messages.MessageField('StreamingApplianceSnapshotConfig', 3) |
| streamingComputationTopology = _messages.MessageField('TopologyConfig', 4) |
| workerHarnessPort = _messages.IntegerField(5, variant=_messages.Variant.INT32) |
| |
| |
| class StreamingSideInputLocation(_messages.Message): |
| r"""Identifies the location of a streaming side input. |
| |
| Fields: |
| stateFamily: Identifies the state family where this side input is stored. |
| tag: Identifies the particular side input within the streaming Dataflow |
| job. |
| """ |
| |
| stateFamily = _messages.StringField(1) |
| tag = _messages.StringField(2) |
| |
| |
| class StreamingStageLocation(_messages.Message): |
| r"""Identifies the location of a streaming computation stage, for stage-to- |
| stage communication. |
| |
| Fields: |
| streamId: Identifies the particular stream within the streaming Dataflow |
| job. |
| """ |
| |
| streamId = _messages.StringField(1) |
| |
| |
| class StringList(_messages.Message): |
| r"""A metric value representing a list of strings. |
| |
| Fields: |
| elements: Elements of the list. |
| """ |
| |
| elements = _messages.StringField(1, repeated=True) |
| |
| |
| class StructuredMessage(_messages.Message): |
| r"""A rich message format, including a human readable string, a key for |
| identifying the message, and structured data associated with the message for |
| programmatic consumption. |
| |
| Fields: |
| messageKey: Identifier for this message type. Used by external systems to |
| internationalize or personalize message. |
| messageText: Human-readable version of message. |
| parameters: The structured data associated with this message. |
| """ |
| |
| messageKey = _messages.StringField(1) |
| messageText = _messages.StringField(2) |
| parameters = _messages.MessageField('Parameter', 3, repeated=True) |
| |
| |
| class TaskRunnerSettings(_messages.Message): |
| r"""Taskrunner configuration settings. |
| |
| Fields: |
| alsologtostderr: Whether to also send taskrunner log info to stderr. |
| baseTaskDir: The location on the worker for task-specific subdirectories. |
| baseUrl: The base URL for the taskrunner to use when accessing Google |
| Cloud APIs. When workers access Google Cloud APIs, they logically do so |
| via relative URLs. If this field is specified, it supplies the base URL |
| to use for resolving these relative URLs. The normative algorithm used |
| is defined by RFC 1808, "Relative Uniform Resource Locators". If not |
| specified, the default value is "http://www.googleapis.com/" |
| commandlinesFileName: The file to store preprocessing commands in. |
| continueOnException: Whether to continue taskrunner if an exception is |
| hit. |
| dataflowApiVersion: The API version of endpoint, e.g. "v1b3" |
| harnessCommand: The command to launch the worker harness. |
| languageHint: The suggested backend language. |
| logDir: The directory on the VM to store logs. |
| logToSerialconsole: Whether to send taskrunner log info to Google Compute |
| Engine VM serial console. |
| logUploadLocation: Indicates where to put logs. If this is not specified, |
| the logs will not be uploaded. The supported resource type is: Google |
| Cloud Storage: storage.googleapis.com/{bucket}/{object} |
| bucket.storage.googleapis.com/{object} |
| oauthScopes: The OAuth2 scopes to be requested by the taskrunner in order |
| to access the Cloud Dataflow API. |
| parallelWorkerSettings: The settings to pass to the parallel worker |
| harness. |
| streamingWorkerMainClass: The streaming worker main class name. |
| taskGroup: The UNIX group ID on the worker VM to use for tasks launched by |
| taskrunner; e.g. "wheel". |
| taskUser: The UNIX user ID on the worker VM to use for tasks launched by |
| taskrunner; e.g. "root". |
| tempStoragePrefix: The prefix of the resources the taskrunner should use |
| for temporary storage. The supported resource type is: Google Cloud |
| Storage: storage.googleapis.com/{bucket}/{object} |
| bucket.storage.googleapis.com/{object} |
| vmId: The ID string of the VM. |
| workflowFileName: The file to store the workflow in. |
| """ |
| |
| alsologtostderr = _messages.BooleanField(1) |
| baseTaskDir = _messages.StringField(2) |
| baseUrl = _messages.StringField(3) |
| commandlinesFileName = _messages.StringField(4) |
| continueOnException = _messages.BooleanField(5) |
| dataflowApiVersion = _messages.StringField(6) |
| harnessCommand = _messages.StringField(7) |
| languageHint = _messages.StringField(8) |
| logDir = _messages.StringField(9) |
| logToSerialconsole = _messages.BooleanField(10) |
| logUploadLocation = _messages.StringField(11) |
| oauthScopes = _messages.StringField(12, repeated=True) |
| parallelWorkerSettings = _messages.MessageField('WorkerSettings', 13) |
| streamingWorkerMainClass = _messages.StringField(14) |
| taskGroup = _messages.StringField(15) |
| taskUser = _messages.StringField(16) |
| tempStoragePrefix = _messages.StringField(17) |
| vmId = _messages.StringField(18) |
| workflowFileName = _messages.StringField(19) |
| |
| |
| class TemplateMetadata(_messages.Message): |
| r"""Metadata describing a template. |
| |
| Fields: |
| description: Optional. A description of the template. |
| name: Required. The name of the template. |
| parameters: The parameters for the template. |
| """ |
| |
| description = _messages.StringField(1) |
| name = _messages.StringField(2) |
| parameters = _messages.MessageField('ParameterMetadata', 3, repeated=True) |
| |
| |
| class TopologyConfig(_messages.Message): |
| r"""Global topology of the streaming Dataflow job, including all |
| computations and their sharded locations. |
| |
| Messages: |
| UserStageToComputationNameMapValue: Maps user stage names to stable |
| computation names. |
| |
| Fields: |
| computations: The computations associated with a streaming Dataflow job. |
| dataDiskAssignments: The disks assigned to a streaming Dataflow job. |
| forwardingKeyBits: The size (in bits) of keys that will be assigned to |
| source messages. |
| persistentStateVersion: Version number for persistent state. |
| userStageToComputationNameMap: Maps user stage names to stable computation |
| names. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class UserStageToComputationNameMapValue(_messages.Message): |
| r"""Maps user stage names to stable computation names. |
| |
| Messages: |
| AdditionalProperty: An additional property for a |
| UserStageToComputationNameMapValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type |
| UserStageToComputationNameMapValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a UserStageToComputationNameMapValue |
| object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| computations = _messages.MessageField('ComputationTopology', 1, repeated=True) |
| dataDiskAssignments = _messages.MessageField('DataDiskAssignment', 2, repeated=True) |
| forwardingKeyBits = _messages.IntegerField(3, variant=_messages.Variant.INT32) |
| persistentStateVersion = _messages.IntegerField(4, variant=_messages.Variant.INT32) |
| userStageToComputationNameMap = _messages.MessageField('UserStageToComputationNameMapValue', 5) |
| |
| |
| class TransformSummary(_messages.Message): |
| r"""Description of the type, names/ids, and input/outputs for a transform. |
| |
| Enums: |
| KindValueValuesEnum: Type of transform. |
| |
| Fields: |
| displayData: Transform-specific display data. |
| id: SDK generated id of this transform instance. |
| inputCollectionName: User names for all collection inputs to this |
| transform. |
| kind: Type of transform. |
| name: User provided name for this transform instance. |
| outputCollectionName: User names for all collection outputs to this |
| transform. |
| """ |
| |
| class KindValueValuesEnum(_messages.Enum): |
| r"""Type of transform. |
| |
| Values: |
| UNKNOWN_KIND: Unrecognized transform type. |
| PAR_DO_KIND: ParDo transform. |
| GROUP_BY_KEY_KIND: Group By Key transform. |
| FLATTEN_KIND: Flatten transform. |
| READ_KIND: Read transform. |
| WRITE_KIND: Write transform. |
| CONSTANT_KIND: Constructs from a constant value, such as with Create.of. |
| SINGLETON_KIND: Creates a Singleton view of a collection. |
| SHUFFLE_KIND: Opening or closing a shuffle session, often as part of a |
| GroupByKey. |
| """ |
| UNKNOWN_KIND = 0 |
| PAR_DO_KIND = 1 |
| GROUP_BY_KEY_KIND = 2 |
| FLATTEN_KIND = 3 |
| READ_KIND = 4 |
| WRITE_KIND = 5 |
| CONSTANT_KIND = 6 |
| SINGLETON_KIND = 7 |
| SHUFFLE_KIND = 8 |
| |
| displayData = _messages.MessageField('DisplayData', 1, repeated=True) |
| id = _messages.StringField(2) |
| inputCollectionName = _messages.StringField(3, repeated=True) |
| kind = _messages.EnumField('KindValueValuesEnum', 4) |
| name = _messages.StringField(5) |
| outputCollectionName = _messages.StringField(6, repeated=True) |
| |
| |
| class ValidateResponse(_messages.Message): |
| r"""Response to the validation request. |
| |
| Fields: |
| errorMessage: Will be empty if validation succeeds. |
| """ |
| |
| errorMessage = _messages.StringField(1) |
| |
| |
| class WorkItem(_messages.Message): |
| r"""WorkItem represents basic information about a WorkItem to be executed in |
| the cloud. |
| |
| Fields: |
| configuration: Work item-specific configuration as an opaque blob. |
| id: Identifies this WorkItem. |
| initialReportIndex: The initial index to use when reporting the status of |
| the WorkItem. |
| jobId: Identifies the workflow job this WorkItem belongs to. |
| leaseExpireTime: Time when the lease on this Work will expire. |
| mapTask: Additional information for MapTask WorkItems. |
| packages: Any required packages that need to be fetched in order to |
| execute this WorkItem. |
| projectId: Identifies the cloud project this WorkItem belongs to. |
| reportStatusInterval: Recommended reporting interval. |
| seqMapTask: Additional information for SeqMapTask WorkItems. |
| shellTask: Additional information for ShellTask WorkItems. |
| sourceOperationTask: Additional information for source operation |
| WorkItems. |
| streamingComputationTask: Additional information for |
| StreamingComputationTask WorkItems. |
| streamingConfigTask: Additional information for StreamingConfigTask |
| WorkItems. |
| streamingSetupTask: Additional information for StreamingSetupTask |
| WorkItems. |
| """ |
| |
| configuration = _messages.StringField(1) |
| id = _messages.IntegerField(2) |
| initialReportIndex = _messages.IntegerField(3) |
| jobId = _messages.StringField(4) |
| leaseExpireTime = _messages.StringField(5) |
| mapTask = _messages.MessageField('MapTask', 6) |
| packages = _messages.MessageField('Package', 7, repeated=True) |
| projectId = _messages.StringField(8) |
| reportStatusInterval = _messages.StringField(9) |
| seqMapTask = _messages.MessageField('SeqMapTask', 10) |
| shellTask = _messages.MessageField('ShellTask', 11) |
| sourceOperationTask = _messages.MessageField('SourceOperationRequest', 12) |
| streamingComputationTask = _messages.MessageField('StreamingComputationTask', 13) |
| streamingConfigTask = _messages.MessageField('StreamingConfigTask', 14) |
| streamingSetupTask = _messages.MessageField('StreamingSetupTask', 15) |
| |
| |
| class WorkItemServiceState(_messages.Message): |
| r"""The Dataflow service's idea of the current state of a WorkItem being |
| processed by a worker. |
| |
| Messages: |
| HarnessDataValue: Other data returned by the service, specific to the |
| particular worker harness. |
| |
| Fields: |
| harnessData: Other data returned by the service, specific to the |
| particular worker harness. |
| leaseExpireTime: Time at which the current lease will expire. |
| metricShortId: The short ids that workers should use in subsequent metric |
| updates. Workers should strive to use short ids whenever possible, but |
| it is ok to request the short_id again if a worker lost track of it |
| (e.g. if the worker is recovering from a crash). NOTE: it is possible |
| that the response may have short ids for a subset of the metrics. |
| nextReportIndex: The index value to use for the next report sent by the |
| worker. Note: If the report call fails for whatever reason, the worker |
| should reuse this index for subsequent report attempts. |
| reportStatusInterval: New recommended reporting interval. |
| splitRequest: The progress point in the WorkItem where the Dataflow |
| service suggests that the worker truncate the task. |
| suggestedStopPoint: DEPRECATED in favor of split_request. |
| suggestedStopPosition: Obsolete, always empty. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class HarnessDataValue(_messages.Message): |
| r"""Other data returned by the service, specific to the particular worker |
| harness. |
| |
| Messages: |
| AdditionalProperty: An additional property for a HarnessDataValue |
| object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a HarnessDataValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| harnessData = _messages.MessageField('HarnessDataValue', 1) |
| leaseExpireTime = _messages.StringField(2) |
| metricShortId = _messages.MessageField('MetricShortId', 3, repeated=True) |
| nextReportIndex = _messages.IntegerField(4) |
| reportStatusInterval = _messages.StringField(5) |
| splitRequest = _messages.MessageField('ApproximateSplitRequest', 6) |
| suggestedStopPoint = _messages.MessageField('ApproximateProgress', 7) |
| suggestedStopPosition = _messages.MessageField('Position', 8) |
| |
| |
| class WorkItemStatus(_messages.Message): |
| r"""Conveys a worker's progress through the work described by a WorkItem. |
| |
| Fields: |
| completed: True if the WorkItem was completed (successfully or |
| unsuccessfully). |
| counterUpdates: Worker output counters for this WorkItem. |
| dynamicSourceSplit: See documentation of stop_position. |
| errors: Specifies errors which occurred during processing. If errors are |
| provided, and completed = true, then the WorkItem is considered to have |
| failed. |
| metricUpdates: DEPRECATED in favor of counter_updates. |
| progress: DEPRECATED in favor of reported_progress. |
| reportIndex: The report index. When a WorkItem is leased, the lease will |
| contain an initial report index. When a WorkItem's status is reported |
| to the system, the report should be sent with that report index, and the |
| response will contain the index the worker should use for the next |
| report. Reports received with unexpected index values will be rejected |
| by the service. In order to preserve idempotency, the worker should not |
| alter the contents of a report, even if the worker must submit the same |
| report multiple times before getting back a response. The worker should |
| not submit a subsequent report until the response for the previous |
| report had been received from the service. |
| reportedProgress: The worker's progress through this WorkItem. |
| requestedLeaseDuration: Amount of time the worker requests for its lease. |
| sourceFork: DEPRECATED in favor of dynamic_source_split. |
| sourceOperationResponse: If the work item represented a |
| SourceOperationRequest, and the work is completed, contains the result |
| of the operation. |
| stopPosition: A worker may split an active map task in two parts, |
| "primary" and "residual", continuing to process the primary part and |
| returning the residual part into the pool of available work. This event |
| is called a "dynamic split" and is critical to the dynamic work |
| rebalancing feature. The two obtained sub-tasks are called "parts" of |
| the split. The parts, if concatenated, must represent the same input as |
| would be read by the current task if the split did not happen. The exact |
| way in which the original task is decomposed into the two parts is |
| specified either as a position demarcating them (stop_position), or |
| explicitly as two DerivedSources, if this task consumes a user-defined |
| source type (dynamic_source_split). The "current" task is adjusted as a |
| result of the split: after a task with range [A, B) sends a |
| stop_position update at C, its range is considered to be [A, C), e.g.: * |
| Progress should be interpreted relative to the new range, e.g. "75% |
| completed" means "75% of [A, C) completed" * The worker should interpret |
| proposed_stop_position relative to the new range, e.g. "split at 68%" |
| should be interpreted as "split at 68% of [A, C)". * If the worker |
| chooses to split again using stop_position, only stop_positions in [A, |
| C) will be accepted. * Etc. dynamic_source_split has similar semantics: |
| e.g., if a task with source S splits using dynamic_source_split into {P, |
| R} (where P and R must be together equivalent to S), then subsequent |
| progress and proposed_stop_position should be interpreted relative to P, |
| and in a potential subsequent dynamic_source_split into {P', R'}, P' and |
| R' must be together equivalent to P, etc. |
| totalThrottlerWaitTimeSeconds: Total time the worker spent being throttled |
| by external systems. |
| workItemId: Identifies the WorkItem. |
| """ |
| |
| completed = _messages.BooleanField(1) |
| counterUpdates = _messages.MessageField('CounterUpdate', 2, repeated=True) |
| dynamicSourceSplit = _messages.MessageField('DynamicSourceSplit', 3) |
| errors = _messages.MessageField('Status', 4, repeated=True) |
| metricUpdates = _messages.MessageField('MetricUpdate', 5, repeated=True) |
| progress = _messages.MessageField('ApproximateProgress', 6) |
| reportIndex = _messages.IntegerField(7) |
| reportedProgress = _messages.MessageField('ApproximateReportedProgress', 8) |
| requestedLeaseDuration = _messages.StringField(9) |
| sourceFork = _messages.MessageField('SourceFork', 10) |
| sourceOperationResponse = _messages.MessageField('SourceOperationResponse', 11) |
| stopPosition = _messages.MessageField('Position', 12) |
| totalThrottlerWaitTimeSeconds = _messages.FloatField(13) |
| workItemId = _messages.StringField(14) |
| |
| |
| class WorkerHealthReport(_messages.Message): |
| r"""WorkerHealthReport contains information about the health of a worker. |
| The VM should be identified by the labels attached to the WorkerMessage that |
| this health ping belongs to. |
| |
| Messages: |
| PodsValueListEntry: A PodsValueListEntry object. |
| |
| Fields: |
| msg: A message describing any unusual health reports. |
| pods: The pods running on the worker. See: http://kubernetes.io/v1.1/docs |
| /api-reference/v1/definitions.html#_v1_pod This field is used by the |
| worker to send the status of the indvidual containers running on each |
| worker. |
| reportInterval: The interval at which the worker is sending health |
| reports. The default value of 0 should be interpreted as the field is |
| not being explicitly set by the worker. |
| vmIsBroken: Whether the VM is in a permanently broken state. Broken VMs |
| should be abandoned or deleted ASAP to avoid assigning or completing any |
| work. |
| vmIsHealthy: Whether the VM is currently healthy. |
| vmStartupTime: The time the VM was booted. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class PodsValueListEntry(_messages.Message): |
| r"""A PodsValueListEntry object. |
| |
| Messages: |
| AdditionalProperty: An additional property for a PodsValueListEntry |
| object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a PodsValueListEntry object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| msg = _messages.StringField(1) |
| pods = _messages.MessageField('PodsValueListEntry', 2, repeated=True) |
| reportInterval = _messages.StringField(3) |
| vmIsBroken = _messages.BooleanField(4) |
| vmIsHealthy = _messages.BooleanField(5) |
| vmStartupTime = _messages.StringField(6) |
| |
| |
| class WorkerHealthReportResponse(_messages.Message): |
| r"""WorkerHealthReportResponse contains information returned to the worker |
| in response to a health ping. |
| |
| Fields: |
| reportInterval: A positive value indicates the worker should change its |
| reporting interval to the specified value. The default value of zero |
| means no change in report rate is requested by the server. |
| """ |
| |
| reportInterval = _messages.StringField(1) |
| |
| |
| class WorkerLifecycleEvent(_messages.Message): |
| r"""A report of an event in a worker's lifecycle. The proto contains one |
| event, because the worker is expected to asynchronously send each message |
| immediately after the event. Due to this asynchrony, messages may arrive out |
| of order (or missing), and it is up to the consumer to interpret. The |
| timestamp of the event is in the enclosing WorkerMessage proto. |
| |
| Enums: |
| EventValueValuesEnum: The event being reported. |
| |
| Messages: |
| MetadataValue: Other stats that can accompany an event. E.g. { |
| "downloaded_bytes" : "123456" } |
| |
| Fields: |
| containerStartTime: The start time of this container. All events will |
| report this so that events can be grouped together across container/VM |
| restarts. |
| event: The event being reported. |
| metadata: Other stats that can accompany an event. E.g. { |
| "downloaded_bytes" : "123456" } |
| """ |
| |
| class EventValueValuesEnum(_messages.Enum): |
| r"""The event being reported. |
| |
| Values: |
| UNKNOWN_EVENT: Invalid event. |
| OS_START: The time the VM started. |
| CONTAINER_START: Our container code starts running. Multiple containers |
| could be distinguished with WorkerMessage.labels if desired. |
| NETWORK_UP: The worker has a functional external network connection. |
| STAGING_FILES_DOWNLOAD_START: Started downloading staging files. |
| STAGING_FILES_DOWNLOAD_FINISH: Finished downloading all staging files. |
| SDK_INSTALL_START: For applicable SDKs, started installation of SDK and |
| worker packages. |
| SDK_INSTALL_FINISH: Finished installing SDK. |
| """ |
| UNKNOWN_EVENT = 0 |
| OS_START = 1 |
| CONTAINER_START = 2 |
| NETWORK_UP = 3 |
| STAGING_FILES_DOWNLOAD_START = 4 |
| STAGING_FILES_DOWNLOAD_FINISH = 5 |
| SDK_INSTALL_START = 6 |
| SDK_INSTALL_FINISH = 7 |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class MetadataValue(_messages.Message): |
| r"""Other stats that can accompany an event. E.g. { "downloaded_bytes" : |
| "123456" } |
| |
| Messages: |
| AdditionalProperty: An additional property for a MetadataValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type MetadataValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a MetadataValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| containerStartTime = _messages.StringField(1) |
| event = _messages.EnumField('EventValueValuesEnum', 2) |
| metadata = _messages.MessageField('MetadataValue', 3) |
| |
| |
| class WorkerMessage(_messages.Message): |
| r"""WorkerMessage provides information to the backend about a worker. |
| |
| Messages: |
| LabelsValue: Labels are used to group WorkerMessages. For example, a |
| worker_message about a particular container might have the labels: { |
| "JOB_ID": "2015-04-22", "WORKER_ID": "wordcount-vm-2015..." |
| "CONTAINER_TYPE": "worker", "CONTAINER_ID": "ac1234def"} Label tags |
| typically correspond to Label enum values. However, for ease of |
| development other strings can be used as tags. LABEL_UNSPECIFIED should |
| not be used here. |
| |
| Fields: |
| labels: Labels are used to group WorkerMessages. For example, a |
| worker_message about a particular container might have the labels: { |
| "JOB_ID": "2015-04-22", "WORKER_ID": "wordcount-vm-2015..." |
| "CONTAINER_TYPE": "worker", "CONTAINER_ID": "ac1234def"} Label tags |
| typically correspond to Label enum values. However, for ease of |
| development other strings can be used as tags. LABEL_UNSPECIFIED should |
| not be used here. |
| time: The timestamp of the worker_message. |
| workerHealthReport: The health of a worker. |
| workerLifecycleEvent: Record of worker lifecycle events. |
| workerMessageCode: A worker message code. |
| workerMetrics: Resource metrics reported by workers. |
| workerShutdownNotice: Shutdown notice by workers. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class LabelsValue(_messages.Message): |
| r"""Labels are used to group WorkerMessages. For example, a worker_message |
| about a particular container might have the labels: { "JOB_ID": |
| "2015-04-22", "WORKER_ID": "wordcount-vm-2015..." "CONTAINER_TYPE": |
| "worker", "CONTAINER_ID": "ac1234def"} Label tags typically correspond |
| to Label enum values. However, for ease of development other strings can |
| be used as tags. LABEL_UNSPECIFIED should not be used here. |
| |
| Messages: |
| AdditionalProperty: An additional property for a LabelsValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type LabelsValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a LabelsValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| labels = _messages.MessageField('LabelsValue', 1) |
| time = _messages.StringField(2) |
| workerHealthReport = _messages.MessageField('WorkerHealthReport', 3) |
| workerLifecycleEvent = _messages.MessageField('WorkerLifecycleEvent', 4) |
| workerMessageCode = _messages.MessageField('WorkerMessageCode', 5) |
| workerMetrics = _messages.MessageField('ResourceUtilizationReport', 6) |
| workerShutdownNotice = _messages.MessageField('WorkerShutdownNotice', 7) |
| |
| |
| class WorkerMessageCode(_messages.Message): |
| r"""A message code is used to report status and error messages to the |
| service. The message codes are intended to be machine readable. The service |
| will take care of translating these into user understandable messages if |
| necessary. Example use cases: 1. Worker processes reporting successful |
| startup. 2. Worker processes reporting specific errors (e.g. package |
| staging failure). |
| |
| Messages: |
| ParametersValue: Parameters contains specific information about the code. |
| This is a struct to allow parameters of different types. Examples: 1. |
| For a "HARNESS_STARTED" message parameters might provide the name of |
| the worker and additional data like timing information. 2. For a |
| "GCS_DOWNLOAD_ERROR" parameters might contain fields listing the GCS |
| objects being downloaded and fields containing errors. In general |
| complex data structures should be avoided. If a worker needs to send a |
| specific and complicated data structure then please consider defining a |
| new proto and adding it to the data oneof in WorkerMessageResponse. |
| Conventions: Parameters should only be used for information that isn't |
| typically passed as a label. hostname and other worker identifiers |
| should almost always be passed as labels since they will be included on |
| most messages. |
| |
| Fields: |
| code: The code is a string intended for consumption by a machine that |
| identifies the type of message being sent. Examples: 1. |
| "HARNESS_STARTED" might be used to indicate the worker harness has |
| started. 2. "GCS_DOWNLOAD_ERROR" might be used to indicate an error |
| downloading a GCS file as part of the boot process of one of the |
| worker containers. This is a string and not an enum to make it easy to |
| add new codes without waiting for an API change. |
| parameters: Parameters contains specific information about the code. This |
| is a struct to allow parameters of different types. Examples: 1. For a |
| "HARNESS_STARTED" message parameters might provide the name of the |
| worker and additional data like timing information. 2. For a |
| "GCS_DOWNLOAD_ERROR" parameters might contain fields listing the GCS |
| objects being downloaded and fields containing errors. In general |
| complex data structures should be avoided. If a worker needs to send a |
| specific and complicated data structure then please consider defining a |
| new proto and adding it to the data oneof in WorkerMessageResponse. |
| Conventions: Parameters should only be used for information that isn't |
| typically passed as a label. hostname and other worker identifiers |
| should almost always be passed as labels since they will be included on |
| most messages. |
| """ |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class ParametersValue(_messages.Message): |
| r"""Parameters contains specific information about the code. This is a |
| struct to allow parameters of different types. Examples: 1. For a |
| "HARNESS_STARTED" message parameters might provide the name of the |
| worker and additional data like timing information. 2. For a |
| "GCS_DOWNLOAD_ERROR" parameters might contain fields listing the GCS |
| objects being downloaded and fields containing errors. In general complex |
| data structures should be avoided. If a worker needs to send a specific |
| and complicated data structure then please consider defining a new proto |
| and adding it to the data oneof in WorkerMessageResponse. Conventions: |
| Parameters should only be used for information that isn't typically passed |
| as a label. hostname and other worker identifiers should almost always be |
| passed as labels since they will be included on most messages. |
| |
| Messages: |
| AdditionalProperty: An additional property for a ParametersValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a ParametersValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| code = _messages.StringField(1) |
| parameters = _messages.MessageField('ParametersValue', 2) |
| |
| |
| class WorkerMessageResponse(_messages.Message): |
| r"""A worker_message response allows the server to pass information to the |
| sender. |
| |
| Fields: |
| workerHealthReportResponse: The service's response to a worker's health |
| report. |
| workerMetricsResponse: Service's response to reporting worker metrics |
| (currently empty). |
| workerShutdownNoticeResponse: Service's response to shutdown notice |
| (currently empty). |
| """ |
| |
| workerHealthReportResponse = _messages.MessageField('WorkerHealthReportResponse', 1) |
| workerMetricsResponse = _messages.MessageField('ResourceUtilizationReportResponse', 2) |
| workerShutdownNoticeResponse = _messages.MessageField('WorkerShutdownNoticeResponse', 3) |
| |
| |
| class WorkerPool(_messages.Message): |
| r"""Describes one particular pool of Cloud Dataflow workers to be |
| instantiated by the Cloud Dataflow service in order to perform the |
| computations required by a job. Note that a workflow job may use multiple |
| pools, in order to match the various computational requirements of the |
| various stages of the job. |
| |
| Enums: |
| DefaultPackageSetValueValuesEnum: The default package set to install. |
| This allows the service to select a default set of packages which are |
| useful to worker harnesses written in a particular language. |
| IpConfigurationValueValuesEnum: Configuration for VM IPs. |
| TeardownPolicyValueValuesEnum: Sets the policy for determining when to |
| turndown worker pool. Allowed values are: `TEARDOWN_ALWAYS`, |
| `TEARDOWN_ON_SUCCESS`, and `TEARDOWN_NEVER`. `TEARDOWN_ALWAYS` means |
| workers are always torn down regardless of whether the job succeeds. |
| `TEARDOWN_ON_SUCCESS` means workers are torn down if the job succeeds. |
| `TEARDOWN_NEVER` means the workers are never torn down. If the workers |
| are not torn down by the service, they will continue to run and use |
| Google Compute Engine VM resources in the user's project until they are |
| explicitly terminated by the user. Because of this, Google recommends |
| using the `TEARDOWN_ALWAYS` policy except for small, manually supervised |
| test jobs. If unknown or unspecified, the service will attempt to |
| choose a reasonable default. |
| |
| Messages: |
| MetadataValue: Metadata to set on the Google Compute Engine VMs. |
| PoolArgsValue: Extra arguments for this worker pool. |
| |
| Fields: |
| autoscalingSettings: Settings for autoscaling of this WorkerPool. |
| dataDisks: Data disks that are used by a VM in this workflow. |
| defaultPackageSet: The default package set to install. This allows the |
| service to select a default set of packages which are useful to worker |
| harnesses written in a particular language. |
| diskSizeGb: Size of root disk for VMs, in GB. If zero or unspecified, the |
| service will attempt to choose a reasonable default. |
| diskSourceImage: Fully qualified source image for disks. |
| diskType: Type of root disk for VMs. If empty or unspecified, the service |
| will attempt to choose a reasonable default. |
| ipConfiguration: Configuration for VM IPs. |
| kind: The kind of the worker pool; currently only `harness` and `shuffle` |
| are supported. |
| machineType: Machine type (e.g. "n1-standard-1"). If empty or |
| unspecified, the service will attempt to choose a reasonable default. |
| metadata: Metadata to set on the Google Compute Engine VMs. |
| network: Network to which VMs will be assigned. If empty or unspecified, |
| the service will use the network "default". |
| numThreadsPerWorker: The number of threads per worker harness. If empty or |
| unspecified, the service will choose a number of threads (according to |
| the number of cores on the selected machine type for batch, or 1 by |
| convention for streaming). |
| numWorkers: Number of Google Compute Engine workers in this pool needed to |
| execute the job. If zero or unspecified, the service will attempt to |
| choose a reasonable default. |
| onHostMaintenance: The action to take on host maintenance, as defined by |
| the Google Compute Engine API. |
| packages: Packages to be installed on workers. |
| poolArgs: Extra arguments for this worker pool. |
| subnetwork: Subnetwork to which VMs will be assigned, if desired. |
| Expected to be of the form "regions/REGION/subnetworks/SUBNETWORK". |
| taskrunnerSettings: Settings passed through to Google Compute Engine |
| workers when using the standard Dataflow task runner. Users should |
| ignore this field. |
| teardownPolicy: Sets the policy for determining when to turndown worker |
| pool. Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and |
| `TEARDOWN_NEVER`. `TEARDOWN_ALWAYS` means workers are always torn down |
| regardless of whether the job succeeds. `TEARDOWN_ON_SUCCESS` means |
| workers are torn down if the job succeeds. `TEARDOWN_NEVER` means the |
| workers are never torn down. If the workers are not torn down by the |
| service, they will continue to run and use Google Compute Engine VM |
| resources in the user's project until they are explicitly terminated by |
| the user. Because of this, Google recommends using the `TEARDOWN_ALWAYS` |
| policy except for small, manually supervised test jobs. If unknown or |
| unspecified, the service will attempt to choose a reasonable default. |
| workerHarnessContainerImage: Required. Docker container image that |
| executes the Cloud Dataflow worker harness, residing in Google Container |
| Registry. |
| zone: Zone to run the worker pools in. If empty or unspecified, the |
| service will attempt to choose a reasonable default. |
| """ |
| |
| class DefaultPackageSetValueValuesEnum(_messages.Enum): |
| r"""The default package set to install. This allows the service to select |
| a default set of packages which are useful to worker harnesses written in |
| a particular language. |
| |
| Values: |
| DEFAULT_PACKAGE_SET_UNKNOWN: The default set of packages to stage is |
| unknown, or unspecified. |
| DEFAULT_PACKAGE_SET_NONE: Indicates that no packages should be staged at |
| the worker unless explicitly specified by the job. |
| DEFAULT_PACKAGE_SET_JAVA: Stage packages typically useful to workers |
| written in Java. |
| DEFAULT_PACKAGE_SET_PYTHON: Stage pacakges typically useful to workers |
| written in Python. |
| """ |
| DEFAULT_PACKAGE_SET_UNKNOWN = 0 |
| DEFAULT_PACKAGE_SET_NONE = 1 |
| DEFAULT_PACKAGE_SET_JAVA = 2 |
| DEFAULT_PACKAGE_SET_PYTHON = 3 |
| |
| class IpConfigurationValueValuesEnum(_messages.Enum): |
| r"""Configuration for VM IPs. |
| |
| Values: |
| WORKER_IP_UNSPECIFIED: The configuration is unknown, or unspecified. |
| WORKER_IP_PUBLIC: Workers should have public IP addresses. |
| WORKER_IP_PRIVATE: Workers should have private IP addresses. |
| """ |
| WORKER_IP_UNSPECIFIED = 0 |
| WORKER_IP_PUBLIC = 1 |
| WORKER_IP_PRIVATE = 2 |
| |
| class TeardownPolicyValueValuesEnum(_messages.Enum): |
| r"""Sets the policy for determining when to turndown worker pool. Allowed |
| values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and |
| `TEARDOWN_NEVER`. `TEARDOWN_ALWAYS` means workers are always torn down |
| regardless of whether the job succeeds. `TEARDOWN_ON_SUCCESS` means |
| workers are torn down if the job succeeds. `TEARDOWN_NEVER` means the |
| workers are never torn down. If the workers are not torn down by the |
| service, they will continue to run and use Google Compute Engine VM |
| resources in the user's project until they are explicitly terminated by |
| the user. Because of this, Google recommends using the `TEARDOWN_ALWAYS` |
| policy except for small, manually supervised test jobs. If unknown or |
| unspecified, the service will attempt to choose a reasonable default. |
| |
| Values: |
| TEARDOWN_POLICY_UNKNOWN: The teardown policy isn't specified, or is |
| unknown. |
| TEARDOWN_ALWAYS: Always teardown the resource. |
| TEARDOWN_ON_SUCCESS: Teardown the resource on success. This is useful |
| for debugging failures. |
| TEARDOWN_NEVER: Never teardown the resource. This is useful for |
| debugging and development. |
| """ |
| TEARDOWN_POLICY_UNKNOWN = 0 |
| TEARDOWN_ALWAYS = 1 |
| TEARDOWN_ON_SUCCESS = 2 |
| TEARDOWN_NEVER = 3 |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class MetadataValue(_messages.Message): |
| r"""Metadata to set on the Google Compute Engine VMs. |
| |
| Messages: |
| AdditionalProperty: An additional property for a MetadataValue object. |
| |
| Fields: |
| additionalProperties: Additional properties of type MetadataValue |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a MetadataValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A string attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.StringField(2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| @encoding.MapUnrecognizedFields('additionalProperties') |
| class PoolArgsValue(_messages.Message): |
| r"""Extra arguments for this worker pool. |
| |
| Messages: |
| AdditionalProperty: An additional property for a PoolArgsValue object. |
| |
| Fields: |
| additionalProperties: Properties of the object. Contains field @type |
| with type URL. |
| """ |
| |
| class AdditionalProperty(_messages.Message): |
| r"""An additional property for a PoolArgsValue object. |
| |
| Fields: |
| key: Name of the additional property. |
| value: A extra_types.JsonValue attribute. |
| """ |
| |
| key = _messages.StringField(1) |
| value = _messages.MessageField('extra_types.JsonValue', 2) |
| |
| additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True) |
| |
| autoscalingSettings = _messages.MessageField('AutoscalingSettings', 1) |
| dataDisks = _messages.MessageField('Disk', 2, repeated=True) |
| defaultPackageSet = _messages.EnumField('DefaultPackageSetValueValuesEnum', 3) |
| diskSizeGb = _messages.IntegerField(4, variant=_messages.Variant.INT32) |
| diskSourceImage = _messages.StringField(5) |
| diskType = _messages.StringField(6) |
| ipConfiguration = _messages.EnumField('IpConfigurationValueValuesEnum', 7) |
| kind = _messages.StringField(8) |
| machineType = _messages.StringField(9) |
| metadata = _messages.MessageField('MetadataValue', 10) |
| network = _messages.StringField(11) |
| numThreadsPerWorker = _messages.IntegerField(12, variant=_messages.Variant.INT32) |
| numWorkers = _messages.IntegerField(13, variant=_messages.Variant.INT32) |
| onHostMaintenance = _messages.StringField(14) |
| packages = _messages.MessageField('Package', 15, repeated=True) |
| poolArgs = _messages.MessageField('PoolArgsValue', 16) |
| subnetwork = _messages.StringField(17) |
| taskrunnerSettings = _messages.MessageField('TaskRunnerSettings', 18) |
| teardownPolicy = _messages.EnumField('TeardownPolicyValueValuesEnum', 19) |
| workerHarnessContainerImage = _messages.StringField(20) |
| zone = _messages.StringField(21) |
| |
| |
| class WorkerSettings(_messages.Message): |
| r"""Provides data to pass through to the worker harness. |
| |
| Fields: |
| baseUrl: The base URL for accessing Google Cloud APIs. When workers |
| access Google Cloud APIs, they logically do so via relative URLs. If |
| this field is specified, it supplies the base URL to use for resolving |
| these relative URLs. The normative algorithm used is defined by RFC |
| 1808, "Relative Uniform Resource Locators". If not specified, the |
| default value is "http://www.googleapis.com/" |
| reportingEnabled: Whether to send work progress updates to the service. |
| servicePath: The Cloud Dataflow service path relative to the root URL, for |
| example, "dataflow/v1b3/projects". |
| shuffleServicePath: The Shuffle service path relative to the root URL, for |
| example, "shuffle/v1beta1". |
| tempStoragePrefix: The prefix of the resources the system should use for |
| temporary storage. The supported resource type is: Google Cloud |
| Storage: storage.googleapis.com/{bucket}/{object} |
| bucket.storage.googleapis.com/{object} |
| workerId: The ID of the worker running this pipeline. |
| """ |
| |
| baseUrl = _messages.StringField(1) |
| reportingEnabled = _messages.BooleanField(2) |
| servicePath = _messages.StringField(3) |
| shuffleServicePath = _messages.StringField(4) |
| tempStoragePrefix = _messages.StringField(5) |
| workerId = _messages.StringField(6) |
| |
| |
| class WorkerShutdownNotice(_messages.Message): |
| r"""Shutdown notification from workers. This is to be sent by the shutdown |
| script of the worker VM so that the backend knows that the VM is being shut |
| down. |
| |
| Fields: |
| reason: The reason for the worker shutdown. Current possible values are: |
| "UNKNOWN": shutdown reason is unknown. "PREEMPTION": shutdown reason |
| is preemption. Other possible reasons may be added in the future. |
| """ |
| |
| reason = _messages.StringField(1) |
| |
| |
| class WorkerShutdownNoticeResponse(_messages.Message): |
| r"""Service-side response to WorkerMessage issuing shutdown notice.""" |
| |
| |
| class WriteInstruction(_messages.Message): |
| r"""An instruction that writes records. Takes one input, produces no |
| outputs. |
| |
| Fields: |
| input: The input. |
| sink: The sink to write to. |
| """ |
| |
| input = _messages.MessageField('InstructionInput', 1) |
| sink = _messages.MessageField('Sink', 2) |
| |
| |
| encoding.AddCustomJsonFieldMapping( |
| StandardQueryParameters, 'f__xgafv', '$.xgafv') |
| encoding.AddCustomJsonEnumMapping( |
| StandardQueryParameters.FXgafvValueValuesEnum, '_1', '1') |
| encoding.AddCustomJsonEnumMapping( |
| StandardQueryParameters.FXgafvValueValuesEnum, '_2', '2') |