blob: 5edeaead0d23478ced6d758550b2771414b1c8dc [file] [log] [blame]
syntax = "proto2";
package heron.proto.api;
option java_package = "org.apache.heron.api.generated";
option java_outer_classname = "TopologyAPI";
// This file defines the logic plan of a topology, including
// components definition, stream schema and others.
// 1. Heron usbmitter pushes the message Topology to state manager at node: topologies/{topology_name}
// 2. When TMaster first time starts, it reads Topology from state manager at node topologies/{topology_name},
// 3. TMaster constructs and distribute PhysicalPlan basing on Topology and StrMgrHelloRequest.
// It also pushes the PhysicalPlan to state manager at node: pplans/{topology_name}
//
// Note:
// 1. message PhysicalPlan also contains a copy of message Topolgoy. We distinguish them:
// - topologies/{topology_name} consist of the topology logic plan first submitted by user,
// it shall only be used to construct the PhysicalPlan when TMaster first time starts
// - pplans/{topology_name} reflects the dynamic state of the topology. Initially, it shall be the same
// as what user has submitted.
// Any runtime changes on Topology in are made to Topology inside the node pplans/{topology_name}.
// For instance, change of TopologyState in runtime will be made in the Topology inside the node pplans/{topology_name}.
enum Grouping {
SHUFFLE = 1;
FIELDS = 2;
ALL = 3;
LOWEST = 4;
NONE = 5;
DIRECT = 6;
CUSTOM = 7;
}
enum Type {
OBJECT = 1; // generic type
}
enum ComponentObjectSpec {
JAVA_CLASS_NAME = 1;
JAVA_SERIALIZED_OBJECT = 2;
PYTHON_CLASS_NAME = 3;
CPP_CLASS_INFO = 4;
}
enum CustomGroupingObjectType {
JAVA_OBJECT = 1;
PYTHON_OBJECT = 2;
}
enum ConfigValueType {
STRING_VALUE = 1; //normal string
JAVA_SERIALIZED_VALUE = 2;
PYTHON_SERIALIZED_VALUE = 3;
}
message CPPClassInfo {
// The extern "C" style function name that can
// instantiate an object of the component
required string class_constructor = 1;
}
message StreamSchema {
message KeyType {
required string key = 1;
required Type type = 2;
}
repeated KeyType keys = 1;
}
message StreamId {
required string id = 1;
required string component_name = 2; // the guy who produced it
}
message OutputStream {
required StreamId stream = 1;
required StreamSchema schema = 2;
}
message InputStream {
required StreamId stream = 1;
required Grouping gtype = 2;
optional StreamSchema grouping_fields = 3;
// we should refactor when upgrading to protobuf v3.0 (#1181)
optional bytes custom_grouping_object = 4;
optional CustomGroupingObjectType type = 5;
}
message Config {
message KeyValue {
required string key = 1;
// Only one of the below are set
optional string value = 2;
optional bytes serialized_value = 3;
// This is made optional for backward compatibility (#1179)
optional ConfigValueType type = 4;
}
repeated KeyValue kvs = 1;
}
message Component {
// The name of the component. Like Tail-FlatMap
required string name = 1;
// Only one of the below two is set, whose meaning is determined
// by the value of spec. For example, when spec is JAVA_CLASS_NAME,
// `class_name` is set and contains Java's class name.
// When spec is JAVA_SERIALIZED_OBJECT, `serialized_object` is set and
// contains a serialized Java object.
optional string class_name = 2;
optional bytes serialized_object = 4;
required Config config = 3;
// This is made optional for backward compatibility
optional ComponentObjectSpec spec = 5;
// present if spec is CPP_CLASS_INFO
optional CPPClassInfo cpp_class_info = 6;
}
message Spout {
required Component comp = 1;
repeated OutputStream outputs = 2;
}
message Bolt {
required Component comp = 1;
repeated InputStream inputs = 2;
repeated OutputStream outputs = 3;
}
enum TopologyState {
RUNNING = 1;
PAUSED = 2;
KILLED = 3;
}
message Topology {
required string id = 1;
required string name = 2;
repeated Spout spouts = 3;
repeated Bolt bolts = 4;
required TopologyState state = 5;
optional Config topology_config = 6;
}