| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| |
| namespace cpp impala |
| namespace java org.apache.impala.thrift |
| |
| include "Status.thrift" |
| include "Types.thrift" |
| |
| enum StatestoreServiceVersion { |
| V1 = 0 |
| } |
| |
| // Structure serialized for the topic AdmissionController::IMPALA_REQUEST_QUEUE_TOPIC. |
| // Statistics for a single admission control pool. The topic key is of the form |
| // "<pool_name>!<backend_id>". |
| struct TPoolStats { |
| // The current number of requests admitted by this host's admission controller |
| // and are currently running. This is an instantaneous value (as opposed to a |
| // cumulative sum). |
| 1: required i64 num_admitted_running; |
| |
| // The current number of queued requests. This is an instantaneous value. |
| 2: required i64 num_queued; |
| |
| // The memory (in bytes) currently reserved on this backend for use by queries in this |
| // pool. E.g. when a query which reserves 10G/host admitted to this pool begins |
| // execution on this impalad, this value increases by 10G. Any other impalads executing |
| // this query will also increment their backend_mem_reserved by 10G. |
| 3: required i64 backend_mem_reserved; |
| } |
| |
| // Structure to describe an executor group. We use this to configure the executor group |
| // for backends during startup and during cluster membership management. |
| struct TExecutorGroupDesc { |
| // The name of the executor group. |
| 1: required string name; |
| // The minimum size of the executor group to be considered healthy. |
| 2: required i64 min_size; |
| } |
| |
| // Structure serialised in the Impala backend topic. Each Impalad |
| // constructs one TBackendDescriptor, and registers it in the cluster-membership |
| // topic. Impalads subscribe to this topic to learn of the location of |
| // all other Impalads in the cluster. Impalads can act as coordinators, executors or |
| // both. |
| struct TBackendDescriptor { |
| // Network address of the thrift based ImpalaInternalService on this backend |
| 1: required Types.TNetworkAddress address; |
| |
| // IP address corresponding to address.hostname. Explicitly including this saves the |
| // cost of resolution at every Impalad (since IP addresses are needed for scheduling) |
| 2: required string ip_address; |
| |
| // True if this is a coordinator node |
| 3: required bool is_coordinator; |
| |
| // True if this is an executor node |
| 4: required bool is_executor; |
| |
| // The address of the debug HTTP server |
| 5: optional Types.TNetworkAddress debug_http_address; |
| |
| // True if the debug webserver is secured (for correctly generating links) |
| 6: optional bool secure_webserver; |
| |
| // IP address + port of KRPC based ImpalaInternalService on this backend |
| 7: optional Types.TNetworkAddress krpc_address; |
| |
| // The amount of memory that can be admitted to this backend (in bytes). |
| 8: required i64 admit_mem_limit; |
| |
| // True if fragment instances should not be scheduled on this daemon because the |
| // daemon has been quiescing, e.g. if it shutting down. |
| 9: required bool is_quiescing; |
| |
| // The list of executor groups that this backend belongs to. Only valid if is_executor |
| // is set, and currently must contain exactly one entry. |
| 10: required list<TExecutorGroupDesc> executor_groups; |
| |
| // The number of admission slots for this backend that can be occupied by running |
| // queries. |
| 11: required i64 admission_slots; |
| } |
| |
| // Description of a single entry in a topic |
| struct TTopicItem { |
| // Human-readable topic entry identifier |
| 1: required string key; |
| |
| // Byte-string value for this topic entry. May not be null-terminated (in that it may |
| // contain null bytes). It can be non-empty when deleted is true. This is needed when |
| // subscribers require additional information not captured in the item key to process |
| // the deleted item (e.g., catalog version of deleted catalog object). |
| 2: required string value; |
| |
| // If true, this item was deleted. When false, this TTopicItem need not be included in |
| // non-delta TTopicDelta's since the latest version of every still-present topic item |
| // will be included. |
| 3: required bool deleted = false; |
| } |
| |
| // Set of changes to a single topic, sent from the statestore to a subscriber as well as |
| // from a subscriber to the statestore. |
| struct TTopicDelta { |
| // Name of the topic this delta applies to |
| 1: required string topic_name; |
| |
| // When is_delta=true, a list of changes to topic entries, including deletions, within |
| // [from_version, to_version]. |
| // When is_delta=false, this is the list of all non-delete topic entries for |
| // [0, to_version], which can be used to reconstruct the topic from scratch. |
| 2: required list<TTopicItem> topic_entries; |
| |
| // True if entries / deletions are relative to the topic at versions [0, from_version]. |
| 3: required bool is_delta; |
| |
| // The Topic version range this delta covers. If there have been changes to the topic, |
| // the update will include all changes in the range: [from_version, to_version). |
| // If there have been no changes in the topic the from_version will match the |
| // to_version. The from_version will always be 0 for non-delta updates. |
| // If this is an update being sent from a subscriber to the statestore, the from_version |
| // is set only when recovering from an inconsistent state, to the last version of the |
| // topic the subscriber successfully processed. The value of to_version doesn't depend |
| // on whether the update is delta or not. |
| 4: optional i64 from_version |
| 5: optional i64 to_version |
| |
| // The minimum topic version of all subscribers to the topic. This can be used to |
| // determine when all subscribers have successfully processed a specific update. |
| // This is guaranteed because no subscriber will ever be sent a topic containing |
| // keys with a version < min_subscriber_topic_version. Only used when sending an update |
| // from the statestore to a subscriber. |
| 6: optional i64 min_subscriber_topic_version |
| |
| // If set and true the statestore must clear the existing topic entries (if any) before |
| // applying the entries in topic_entries. |
| 7: optional bool clear_topic_entries |
| } |
| |
| // Description of a topic to subscribe to as part of a RegisterSubscriber call |
| struct TTopicRegistration { |
| // Human readable key for this topic |
| 1: required string topic_name; |
| |
| // True if updates to this topic from this subscriber should be removed upon the |
| // subscriber's failure or disconnection |
| 2: required bool is_transient; |
| |
| // If true, min_subscriber_topic_version is computed and set in topic updates sent |
| // to this subscriber to this subscriber. Should only be set to true if this is |
| // actually required - computing the version is relatively expensive compared to |
| // other aspects of preparing topic updates - see IMPALA-6816. |
| 3: required bool populate_min_subscriber_topic_version = false; |
| |
| // Restrict the items to receive on this subscription to only those items |
| // starting with the given prefix. |
| // |
| // If this is not specified, all items will be subscribed to. |
| 4: optional string filter_prefix |
| } |
| |
| struct TRegisterSubscriberRequest { |
| 1: required StatestoreServiceVersion protocol_version = |
| StatestoreServiceVersion.V1 |
| |
| // Unique, human-readable identifier for this subscriber |
| 2: required string subscriber_id; |
| |
| // Location of the StatestoreSubscriberService that this subscriber runs |
| 3: required Types.TNetworkAddress subscriber_location; |
| |
| // List of topics to subscribe to |
| 4: required list<TTopicRegistration> topic_registrations; |
| } |
| |
| struct TRegisterSubscriberResponse { |
| // Whether the call was executed correctly at the application level |
| 1: required Status.TStatus status; |
| |
| // Unique identifier for this registration. Changes with every call to |
| // RegisterSubscriber(). |
| 2: optional Types.TUniqueId registration_id; |
| } |
| |
| service StatestoreService { |
| // Register a single subscriber. Note that after a subscriber is registered, no new |
| // topics may be added. |
| TRegisterSubscriberResponse RegisterSubscriber(1: TRegisterSubscriberRequest params); |
| } |
| |
| struct TUpdateStateRequest { |
| 1: required StatestoreServiceVersion protocol_version = |
| StatestoreServiceVersion.V1 |
| |
| // Map from topic name to a list of changes for that topic. |
| 2: required map<string, TTopicDelta> topic_deltas; |
| |
| // Registration ID for the last known registration from this subscriber. |
| 3: optional Types.TUniqueId registration_id; |
| } |
| |
| struct TUpdateStateResponse { |
| // Whether the call was executed correctly at the application level |
| 1: required Status.TStatus status; |
| |
| // List of updates published by the subscriber to be made centrally by the statestore |
| 2: required list<TTopicDelta> topic_updates; |
| |
| // True if this update was skipped by the subscriber. This is distinguished from a |
| // non-OK status since the former indicates an error which contributes to the |
| // statestore's view of a subscriber's liveness. |
| 3: optional bool skipped; |
| } |
| |
| struct THeartbeatRequest { |
| 1: optional Types.TUniqueId registration_id; |
| } |
| |
| struct THeartbeatResponse { |
| |
| } |
| |
| service StatestoreSubscriber { |
| // Called when the statestore sends a topic update. The request contains a map of |
| // topic names to TTopicDelta updates, sent from the statestore to the subscriber. Each |
| // of these delta updates will contain a list of additions to the topic and a list of |
| // deletions from the topic. |
| // In response, the subscriber returns an aggregated list of updates to topic(s) to |
| // the statestore. Each update is a TTopicDelta that contains a list of additions to |
| // the topic and a list of deletions from the topic. Additionally, if a subscriber has |
| // received an unexpected delta update version range, they can request a new delta |
| // update based off a specific version from the statestore. The next statestore |
| // delta update will be based off of the version the subscriber requested. |
| TUpdateStateResponse UpdateState(1: TUpdateStateRequest params); |
| |
| // Called when the statestore sends a heartbeat. |
| THeartbeatResponse Heartbeat(1: THeartbeatRequest params); |
| } |