proto/src/main/proto/Rss.proto - incubator-uniffle - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 syntax = "proto3";
 import "google/protobuf/empty.proto";
 import "google/protobuf/wrappers.proto";
 option java_package = "org.apache.uniffle.proto";
 option java_outer_classname = "RssProtos";
 option java_generate_equals_and_hash = true;
 package rss.common;

 service ShuffleServer {
   rpc registerShuffle (ShuffleRegisterRequest) returns (ShuffleRegisterResponse);
   rpc sendShuffleData (SendShuffleDataRequest) returns (SendShuffleDataResponse);
   rpc getLocalShuffleIndex (GetLocalShuffleIndexRequest) returns (GetLocalShuffleIndexResponse);
   rpc getLocalShuffleData (GetLocalShuffleDataRequest) returns (GetLocalShuffleDataResponse);
   rpc getMemoryShuffleData (GetMemoryShuffleDataRequest) returns (GetMemoryShuffleDataResponse);
   rpc commitShuffleTask (ShuffleCommitRequest) returns (ShuffleCommitResponse);
   rpc reportShuffleResult (ReportShuffleResultRequest) returns (ReportShuffleResultResponse);
   rpc getShuffleResult (GetShuffleResultRequest) returns (GetShuffleResultResponse);
   rpc getShuffleResultForMultiPart (GetShuffleResultForMultiPartRequest) returns (GetShuffleResultForMultiPartResponse);
   rpc finishShuffle (FinishShuffleRequest) returns (FinishShuffleResponse);
   rpc requireBuffer (RequireBufferRequest) returns (RequireBufferResponse);
   rpc appHeartbeat(AppHeartBeatRequest) returns (AppHeartBeatResponse);
 }

 message FinishShuffleRequest {
   string appId = 1;
   int32 shuffleId = 2;
 }

 message FinishShuffleResponse {
   StatusCode status = 1;
   string retMsg = 2;
 }

 message RequireBufferRequest {
   int32 requireSize = 1;
 }

 message RequireBufferResponse {
   int64 requireBufferId = 1;
   StatusCode status = 2;
   string retMsg = 3;
 }

 message ShuffleDataBlockSegment {
   int64 blockId = 1;
   int64 offset = 2;
   int32 length = 3;
   int32 uncompressLength = 4;
   int64 crc = 5;
   int64 taskAttemptId = 6;
 }

 message GetLocalShuffleDataRequest {
   string appId = 1;
   int32 shuffleId = 2;
   int32 partitionId = 3;
   int32 partitionNumPerRange = 4;
   int32 partitionNum = 5;
   int64 offset = 6;
   int32 length = 7;
 }

 message GetLocalShuffleDataResponse {
   bytes data = 1;
   StatusCode status = 2;
   string retMsg = 3;
 }

 message GetMemoryShuffleDataRequest {
   string appId = 1;
   int32 shuffleId = 2;
   int32 partitionId = 3;
   int64 lastBlockId = 4;
   int32 readBufferSize = 5;
 }

 message GetMemoryShuffleDataResponse {
   repeated ShuffleDataBlockSegment shuffleDataBlockSegments = 1;
   bytes data = 2;
   StatusCode status = 3;
   string retMsg = 4;
 }

 message GetLocalShuffleIndexRequest {
   string appId = 1;
   int32 shuffleId = 2;
   int32 partitionId = 3;
   int32 partitionNumPerRange = 4;
   int32 partitionNum = 5;
 }

 message GetLocalShuffleIndexResponse {
   bytes indexData = 1;
   StatusCode status = 2;
   string retMsg = 3;
 }

 message ReportShuffleResultRequest {
   string appId = 1;
   int32 shuffleId = 2;
   int64 taskAttemptId = 3;
   int32 bitmapNum = 4;
   repeated PartitionToBlockIds partitionToBlockIds = 5;
 }

 message PartitionToBlockIds {
   int32 partitionId = 1;
   repeated int64 blockIds = 2;
 }

 message ReportShuffleResultResponse {
   StatusCode status = 1;
   string retMsg = 2;
 }

 message GetShuffleResultRequest {
   string appId = 1;
   int32 shuffleId = 2;
   int32 partitionId = 3;
 }

 message GetShuffleResultResponse {
   StatusCode status = 1;
   string retMsg = 2;
   bytes serializedBitmap = 3;
 }

 message GetShuffleResultForMultiPartRequest {
   string appId = 1;
   int32 shuffleId = 2;
   repeated int32 partitions = 3;
 }

 message GetShuffleResultForMultiPartResponse {
   StatusCode status = 1;
   string retMsg = 2;
   bytes serializedBitmap = 3;
 }

 message ShufflePartitionRange {
   int32 start = 1;
   int32 end = 2;
 }

 message ShuffleRegisterRequest {
   string appId = 1;
   int32 shuffleId = 2;
   repeated ShufflePartitionRange partitionRanges = 3;
   RemoteStorage remoteStorage = 4;
   string user = 5;
 }

 message ShuffleRegisterResponse {
   StatusCode status = 1;
   string retMsg = 2;
 }

 message SendShuffleDataRequest {
   string appId = 1;
   int32 shuffleId = 2;
   int64 requireBufferId = 3;
   repeated ShuffleData shuffleData = 4;
 }

 message SendShuffleDataResponse {
   StatusCode status = 1;
   string retMsg = 2;
 }

 message ShuffleData {
   int32 partitionId = 1;
   repeated ShuffleBlock block = 2;
 }

 message ShuffleBlock {
   int64 blockId = 1;
   int32 length = 2;
   int32 uncompressLength = 3;
   int64 crc = 4;
   bytes data = 5;
   int64 taskAttemptId = 6;
 }

 message ShuffleCommitRequest {
   string appId = 1;
   int32 shuffleId = 2;
 }

 message ShuffleCommitResponse {
   int32 commitCount = 1;
   StatusCode status = 2;
   string retMsg = 3;
 }

 message ShuffleServerHeartBeatRequest {
   ShuffleServerId serverId = 1;
   int64 usedMemory = 2;
   int64 preAllocatedMemory = 3;
   int64 availableMemory = 4;
   int32 eventNumInFlush = 5;
   repeated string tags = 6;
   google.protobuf.BoolValue isHealthy = 7;
 }

 message ShuffleServerHeartBeatResponse {
   StatusCode status = 1;
   string retMsg = 2;
 }

 message ShuffleServerId {
   string id = 1;
   string ip = 2;
   int32 port = 3;
 }

 message ShuffleServerResult {
   StatusCode status = 1;
   string retMsg = 2;
 }

 /** Status code to identify the status of response */
 enum StatusCode {
   SUCCESS = 0;
   DOUBLE_REGISTER = 1;
   NO_BUFFER = 2;
   INVALID_STORAGE = 3;
   NO_REGISTER = 4;
   NO_PARTITION = 5;
   INTERNAL_ERROR = 6;
   TIMEOUT = 7;
   ACCESS_DENIED = 8;
   // add more status
 }

 service CoordinatorServer {
   // Get Shuffle Server list
   rpc getShuffleServerList(google.protobuf.Empty) returns (GetShuffleServerListResponse);

   // Count Shuffle Server number
   rpc getShuffleServerNum(google.protobuf.Empty) returns (GetShuffleServerNumResponse);

   // Ask for suitable Shuffle Servers with partitions
   rpc getShuffleAssignments(GetShuffleServerRequest) returns (GetShuffleAssignmentsResponse);

   // Heartbeat between Shuffle Server and Coordinator Server
   rpc heartbeat(ShuffleServerHeartBeatRequest) returns (ShuffleServerHeartBeatResponse);

   // Get the global configuration of this Rss-cluster, i.e., data storage info
   rpc getShuffleDataStorageInfo(google.protobuf.Empty) returns (GetShuffleDataStorageInfoResponse);
   rpc checkServiceAvailable(google.protobuf.Empty) returns (CheckServiceAvailableResponse);

   // Report a client operation's result to coordinator server
   rpc reportClientOperation(ReportShuffleClientOpRequest) returns (ReportShuffleClientOpResponse);

   // Heartbeat between Shuffle Application and Coordinator Server
   rpc appHeartbeat(AppHeartBeatRequest) returns (AppHeartBeatResponse);

   // Access to the remote shuffle service cluster
   rpc accessCluster(AccessClusterRequest) returns (AccessClusterResponse);

   // Get basic client conf from coordinator
   rpc fetchClientConf(google.protobuf.Empty) returns (FetchClientConfResponse);

   // Get remote storage from coordinator
   rpc fetchRemoteStorage(FetchRemoteStorageRequest) returns (FetchRemoteStorageResponse);
 }

 message AppHeartBeatRequest {
   string appId = 1;
 }

 message AppHeartBeatResponse {
   StatusCode status = 1;
   string retMsg = 2;
 }

 message GetShuffleServerListResponse {
   repeated ShuffleServerId servers = 1;
 }

 message GetShuffleServerNumResponse {
   int32 num = 1;
 }

 message GetShuffleServerRequest {
   string clientHost = 1;
   string clientPort = 2;
   string clientProperty = 3;
   string applicationId = 4;
   int32 shuffleId = 5;
   int32 partitionNum = 6;
   int32 partitionNumPerRange = 7;
   int32 dataReplica = 8;
   repeated string requireTags = 9;
   int32 assignmentShuffleServerNumber = 10;
 }

 message PartitionRangeAssignment {
   int32 startPartition = 1;
   int32 endPartition = 2;
   // replica
   repeated ShuffleServerId server = 3;
 }

 message GetShuffleAssignmentsResponse {
   StatusCode status = 1;
   repeated PartitionRangeAssignment assignments = 2;
   string retMsg = 3;
 }

 message ReportShuffleClientOpRequest {
   string clientHost = 1;
   int32 clientPort = 2;
   ShuffleServerId server = 3;
   string operation = 4;
 }

 message ReportShuffleClientOpResponse {
   StatusCode status = 1;
   string retMsg = 2;
 }

 message GetShuffleDataStorageInfoResponse {
   string storage = 1;
   string storagePath = 2;
   string storagePattern = 3;
 }

 message CheckServiceAvailableResponse {
   StatusCode status = 1;
   bool available = 2;
 }

 message AccessClusterRequest {
   string accessId = 1;
   repeated string tags = 2;
   map<string, string> extraProperties = 3;
 }

 message AccessClusterResponse {
   StatusCode status = 1;
   string retMsg = 2;
 }

 message FetchClientConfResponse {
   StatusCode status = 1;
   string retMsg = 2;
   repeated ClientConfItem clientConf = 3;
 }

 message ClientConfItem {
   string key = 1;
   string value = 2;
 }

 message FetchRemoteStorageRequest {
   string appId = 1;
 }

 message RemoteStorageConfItem {
   string key = 1;
   string value = 2;
 }

 message RemoteStorage {
   string path = 1;
   repeated RemoteStorageConfItem remoteStorageConf = 2;
 }

 message FetchRemoteStorageResponse {
   StatusCode status = 1;
   RemoteStorage remoteStorage = 2;
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	syntax = "proto3";
	import "google/protobuf/empty.proto";
	import "google/protobuf/wrappers.proto";
	option java_package = "org.apache.uniffle.proto";
	option java_outer_classname = "RssProtos";
	option java_generate_equals_and_hash = true;
	package rss.common;

	service ShuffleServer {
	rpc registerShuffle (ShuffleRegisterRequest) returns (ShuffleRegisterResponse);
	rpc sendShuffleData (SendShuffleDataRequest) returns (SendShuffleDataResponse);
	rpc getLocalShuffleIndex (GetLocalShuffleIndexRequest) returns (GetLocalShuffleIndexResponse);
	rpc getLocalShuffleData (GetLocalShuffleDataRequest) returns (GetLocalShuffleDataResponse);
	rpc getMemoryShuffleData (GetMemoryShuffleDataRequest) returns (GetMemoryShuffleDataResponse);
	rpc commitShuffleTask (ShuffleCommitRequest) returns (ShuffleCommitResponse);
	rpc reportShuffleResult (ReportShuffleResultRequest) returns (ReportShuffleResultResponse);
	rpc getShuffleResult (GetShuffleResultRequest) returns (GetShuffleResultResponse);
	rpc getShuffleResultForMultiPart (GetShuffleResultForMultiPartRequest) returns (GetShuffleResultForMultiPartResponse);
	rpc finishShuffle (FinishShuffleRequest) returns (FinishShuffleResponse);
	rpc requireBuffer (RequireBufferRequest) returns (RequireBufferResponse);
	rpc appHeartbeat(AppHeartBeatRequest) returns (AppHeartBeatResponse);
	}

	message FinishShuffleRequest {
	string appId = 1;
	int32 shuffleId = 2;
	}

	message FinishShuffleResponse {
	StatusCode status = 1;
	string retMsg = 2;
	}

	message RequireBufferRequest {
	int32 requireSize = 1;
	}

	message RequireBufferResponse {
	int64 requireBufferId = 1;
	StatusCode status = 2;
	string retMsg = 3;
	}

	message ShuffleDataBlockSegment {
	int64 blockId = 1;
	int64 offset = 2;
	int32 length = 3;
	int32 uncompressLength = 4;
	int64 crc = 5;
	int64 taskAttemptId = 6;
	}

	message GetLocalShuffleDataRequest {
	string appId = 1;
	int32 shuffleId = 2;
	int32 partitionId = 3;
	int32 partitionNumPerRange = 4;
	int32 partitionNum = 5;
	int64 offset = 6;
	int32 length = 7;
	}

	message GetLocalShuffleDataResponse {
	bytes data = 1;
	StatusCode status = 2;
	string retMsg = 3;
	}

	message GetMemoryShuffleDataRequest {
	string appId = 1;
	int32 shuffleId = 2;
	int32 partitionId = 3;
	int64 lastBlockId = 4;
	int32 readBufferSize = 5;
	}

	message GetMemoryShuffleDataResponse {
	repeated ShuffleDataBlockSegment shuffleDataBlockSegments = 1;
	bytes data = 2;
	StatusCode status = 3;
	string retMsg = 4;
	}

	message GetLocalShuffleIndexRequest {
	string appId = 1;
	int32 shuffleId = 2;
	int32 partitionId = 3;
	int32 partitionNumPerRange = 4;
	int32 partitionNum = 5;
	}

	message GetLocalShuffleIndexResponse {
	bytes indexData = 1;
	StatusCode status = 2;
	string retMsg = 3;
	}

	message ReportShuffleResultRequest {
	string appId = 1;
	int32 shuffleId = 2;
	int64 taskAttemptId = 3;
	int32 bitmapNum = 4;
	repeated PartitionToBlockIds partitionToBlockIds = 5;
	}

	message PartitionToBlockIds {
	int32 partitionId = 1;
	repeated int64 blockIds = 2;
	}

	message ReportShuffleResultResponse {
	StatusCode status = 1;
	string retMsg = 2;
	}

	message GetShuffleResultRequest {
	string appId = 1;
	int32 shuffleId = 2;
	int32 partitionId = 3;
	}

	message GetShuffleResultResponse {
	StatusCode status = 1;
	string retMsg = 2;
	bytes serializedBitmap = 3;
	}

	message GetShuffleResultForMultiPartRequest {
	string appId = 1;
	int32 shuffleId = 2;
	repeated int32 partitions = 3;
	}

	message GetShuffleResultForMultiPartResponse {
	StatusCode status = 1;
	string retMsg = 2;
	bytes serializedBitmap = 3;
	}

	message ShufflePartitionRange {
	int32 start = 1;
	int32 end = 2;
	}

	message ShuffleRegisterRequest {
	string appId = 1;
	int32 shuffleId = 2;
	repeated ShufflePartitionRange partitionRanges = 3;
	RemoteStorage remoteStorage = 4;
	string user = 5;
	}

	message ShuffleRegisterResponse {
	StatusCode status = 1;
	string retMsg = 2;
	}

	message SendShuffleDataRequest {
	string appId = 1;
	int32 shuffleId = 2;
	int64 requireBufferId = 3;
	repeated ShuffleData shuffleData = 4;
	}

	message SendShuffleDataResponse {
	StatusCode status = 1;
	string retMsg = 2;
	}

	message ShuffleData {
	int32 partitionId = 1;
	repeated ShuffleBlock block = 2;
	}

	message ShuffleBlock {
	int64 blockId = 1;
	int32 length = 2;
	int32 uncompressLength = 3;
	int64 crc = 4;
	bytes data = 5;
	int64 taskAttemptId = 6;
	}

	message ShuffleCommitRequest {
	string appId = 1;
	int32 shuffleId = 2;
	}

	message ShuffleCommitResponse {
	int32 commitCount = 1;
	StatusCode status = 2;
	string retMsg = 3;
	}

	message ShuffleServerHeartBeatRequest {
	ShuffleServerId serverId = 1;
	int64 usedMemory = 2;
	int64 preAllocatedMemory = 3;
	int64 availableMemory = 4;
	int32 eventNumInFlush = 5;
	repeated string tags = 6;
	google.protobuf.BoolValue isHealthy = 7;
	}

	message ShuffleServerHeartBeatResponse {
	StatusCode status = 1;
	string retMsg = 2;
	}

	message ShuffleServerId {
	string id = 1;
	string ip = 2;
	int32 port = 3;
	}

	message ShuffleServerResult {
	StatusCode status = 1;
	string retMsg = 2;
	}

	/** Status code to identify the status of response */
	enum StatusCode {
	SUCCESS = 0;
	DOUBLE_REGISTER = 1;
	NO_BUFFER = 2;
	INVALID_STORAGE = 3;
	NO_REGISTER = 4;
	NO_PARTITION = 5;
	INTERNAL_ERROR = 6;
	TIMEOUT = 7;
	ACCESS_DENIED = 8;
	// add more status
	}

	service CoordinatorServer {
	// Get Shuffle Server list
	rpc getShuffleServerList(google.protobuf.Empty) returns (GetShuffleServerListResponse);

	// Count Shuffle Server number
	rpc getShuffleServerNum(google.protobuf.Empty) returns (GetShuffleServerNumResponse);

	// Ask for suitable Shuffle Servers with partitions
	rpc getShuffleAssignments(GetShuffleServerRequest) returns (GetShuffleAssignmentsResponse);

	// Heartbeat between Shuffle Server and Coordinator Server
	rpc heartbeat(ShuffleServerHeartBeatRequest) returns (ShuffleServerHeartBeatResponse);

	// Get the global configuration of this Rss-cluster, i.e., data storage info
	rpc getShuffleDataStorageInfo(google.protobuf.Empty) returns (GetShuffleDataStorageInfoResponse);
	rpc checkServiceAvailable(google.protobuf.Empty) returns (CheckServiceAvailableResponse);

	// Report a client operation's result to coordinator server
	rpc reportClientOperation(ReportShuffleClientOpRequest) returns (ReportShuffleClientOpResponse);

	// Heartbeat between Shuffle Application and Coordinator Server
	rpc appHeartbeat(AppHeartBeatRequest) returns (AppHeartBeatResponse);

	// Access to the remote shuffle service cluster
	rpc accessCluster(AccessClusterRequest) returns (AccessClusterResponse);

	// Get basic client conf from coordinator
	rpc fetchClientConf(google.protobuf.Empty) returns (FetchClientConfResponse);

	// Get remote storage from coordinator
	rpc fetchRemoteStorage(FetchRemoteStorageRequest) returns (FetchRemoteStorageResponse);
	}

	message AppHeartBeatRequest {
	string appId = 1;
	}

	message AppHeartBeatResponse {
	StatusCode status = 1;
	string retMsg = 2;
	}

	message GetShuffleServerListResponse {
	repeated ShuffleServerId servers = 1;
	}

	message GetShuffleServerNumResponse {
	int32 num = 1;
	}

	message GetShuffleServerRequest {
	string clientHost = 1;
	string clientPort = 2;
	string clientProperty = 3;
	string applicationId = 4;
	int32 shuffleId = 5;
	int32 partitionNum = 6;
	int32 partitionNumPerRange = 7;
	int32 dataReplica = 8;
	repeated string requireTags = 9;
	int32 assignmentShuffleServerNumber = 10;
	}

	message PartitionRangeAssignment {
	int32 startPartition = 1;
	int32 endPartition = 2;
	// replica
	repeated ShuffleServerId server = 3;
	}

	message GetShuffleAssignmentsResponse {
	StatusCode status = 1;
	repeated PartitionRangeAssignment assignments = 2;
	string retMsg = 3;
	}

	message ReportShuffleClientOpRequest {
	string clientHost = 1;
	int32 clientPort = 2;
	ShuffleServerId server = 3;
	string operation = 4;
	}

	message ReportShuffleClientOpResponse {
	StatusCode status = 1;
	string retMsg = 2;
	}

	message GetShuffleDataStorageInfoResponse {
	string storage = 1;
	string storagePath = 2;
	string storagePattern = 3;
	}

	message CheckServiceAvailableResponse {
	StatusCode status = 1;
	bool available = 2;
	}

	message AccessClusterRequest {
	string accessId = 1;
	repeated string tags = 2;
	map<string, string> extraProperties = 3;
	}

	message AccessClusterResponse {
	StatusCode status = 1;
	string retMsg = 2;
	}

	message FetchClientConfResponse {
	StatusCode status = 1;
	string retMsg = 2;
	repeated ClientConfItem clientConf = 3;
	}

	message ClientConfItem {
	string key = 1;
	string value = 2;
	}

	message FetchRemoteStorageRequest {
	string appId = 1;
	}

	message RemoteStorageConfItem {
	string key = 1;
	string value = 2;
	}

	message RemoteStorage {
	string path = 1;
	repeated RemoteStorageConfItem remoteStorageConf = 2;
	}

	message FetchRemoteStorageResponse {
	StatusCode status = 1;
	RemoteStorage remoteStorage = 2;
	}