heron/proto/topology.proto

//  Licensed to the Apache Software Foundation (ASF) under one
//  or more contributor license agreements.  See the NOTICE file
//  distributed with this work for additional information
//  regarding copyright ownership.  The ASF licenses this file
//  to you under the Apache License, Version 2.0 (the
//  "License"); you may not use this file except in compliance
//  with the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an
//  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
//  KIND, either express or implied.  See the License for the
//  specific language governing permissions and limitations
//  under the License.

syntax = "proto2";
package heron.proto.api;

option java_package = "org.apache.heron.api.generated";
option java_outer_classname = "TopologyAPI";

// This file defines the logic plan of a topology, including
// components definition, stream schema and others.
// 1. Heron usbmitter pushes the message Topology to state manager at node: topologies/{topology_name}
// 2. When TMaster first time starts, it reads Topology from state manager at node topologies/{topology_name},
// 3. TMaster constructs and distribute PhysicalPlan basing on Topology and StrMgrHelloRequest.
//    It also pushes the PhysicalPlan to state manager at node: pplans/{topology_name}
//
// Note:
// 1. message PhysicalPlan also contains a copy of message Topolgoy. We distinguish them:
//    - topologies/{topology_name} consist of the topology logic plan first submitted by user,
//      it shall only be used to construct the PhysicalPlan when TMaster first time starts
//    - pplans/{topology_name} reflects the dynamic state of the topology. Initially, it shall be the same
//      as what user has submitted.
//    Any runtime changes on Topology in are made to Topology inside the node pplans/{topology_name}.
//    For instance, change of TopologyState in runtime will be made in the Topology inside the node pplans/{topology_name}.

enum Grouping {
  SHUFFLE = 1;
  FIELDS = 2;
  ALL = 3;
  LOWEST = 4;
  NONE = 5;
  DIRECT = 6;
  CUSTOM = 7;
}

enum Type {
  OBJECT = 1; // generic type
}

enum ComponentObjectSpec {
  JAVA_CLASS_NAME = 1;
  JAVA_SERIALIZED_OBJECT = 2;
  PYTHON_CLASS_NAME = 3;
  CPP_CLASS_INFO = 4;
}

enum CustomGroupingObjectType {
  JAVA_OBJECT = 1;
  PYTHON_OBJECT = 2;
}

enum ConfigValueType {
  STRING_VALUE = 1; //normal string
  JAVA_SERIALIZED_VALUE = 2;
  PYTHON_SERIALIZED_VALUE = 3;
}

message CPPClassInfo {
  // The extern "C" style function name that can
  // instantiate an object of the component
  required string class_constructor = 1;
}

message StreamSchema {
  message KeyType {
    required string key = 1;
    required Type type = 2;
  }
  repeated KeyType keys = 1;
}

message StreamId {
  required string id = 1;
  required string component_name = 2; // the guy who produced it
}

message OutputStream {
  required StreamId stream = 1;
  required StreamSchema schema = 2;
}

message InputStream {
  required StreamId stream = 1;
  required Grouping gtype = 2;
  optional StreamSchema grouping_fields = 3;

  // we should refactor when upgrading to protobuf v3.0 (#1181)
  optional bytes custom_grouping_object = 4;
  optional CustomGroupingObjectType type = 5;
}

message Config {
  message KeyValue {
    required string key = 1;
    // Only one of the below are set
    optional string value = 2;
    optional bytes serialized_value = 3;

    // This is made optional for backward compatibility (#1179)
    optional ConfigValueType type = 4;
  }
  repeated KeyValue kvs = 1;
}

message Component {
  // The name of the component. Like Tail-FlatMap
  required string name = 1;
  // Only one of the below two is set, whose meaning is determined
  // by the value of spec. For example, when spec is JAVA_CLASS_NAME,
  // `class_name` is set and contains Java's class name.
  // When spec is JAVA_SERIALIZED_OBJECT, `serialized_object` is set and
  // contains a serialized Java object.
  optional string class_name = 2;
  optional bytes serialized_object = 4;

  required Config config = 3;

  // This is made optional for backward compatibility
  optional ComponentObjectSpec spec = 5;

  // present if spec is CPP_CLASS_INFO
  optional CPPClassInfo cpp_class_info = 6;
}

message Spout {
  required Component comp = 1;
  repeated OutputStream outputs = 2;
}

message Bolt {
  required Component comp = 1;
  repeated InputStream inputs = 2;
  repeated OutputStream outputs = 3;
}

enum TopologyState {
  RUNNING = 1;
  PAUSED = 2;
  KILLED = 3;
}

message Topology {
  required string id = 1;
  required string name = 2;
  repeated Spout spouts = 3;
  repeated Bolt bolts = 4;
  required TopologyState state = 5;
  optional Config topology_config = 6;
}