protos/engine/manifest.proto

// Copyright 2023 CeresDB Project Authors. Licensed under Apache-2.0.

// Manifest of engine
syntax = "proto3";
package manifest;

import "cluster.proto";
import "engine/schema.proto";
import "engine/time_range.proto";

// Meta update for a new space
message AddSpaceMeta {
  uint32 space_id = 1;
  string space_name = 2;
}

// Meta update for a new table
message AddTableMeta {
  uint32 space_id = 1;
  uint64 table_id = 2;
  string table_name = 3;
  // Schema of the table
  schema.TableSchema schema = 4;
  // Options of the table
  TableOptions options = 5;
  cluster.PartitionInfo partition_info = 6;
}

// Meta update for dropping a table
message DropTableMeta {
  uint32 space_id = 1;
  uint64 table_id = 2;
  string table_name = 3;
}

// Meta data of a sst file
message AddFileMeta {
  // Level of the file
  uint32 level = 1;
  // Id of the file
  uint64 file_id = 2;
  uint64 max_seq = 3;
  time_range.TimeRange time_range = 4;
  uint64 size = 5;
  uint64 row_num = 6;
  StorageFormat storage_format = 7;
  repeated string associated_files = 8;
}

// Meta data of the file to delete
message DeleteFileMeta {
  // Level of the file
  uint32 level = 1;
  // Id of the file
  uint64 file_id = 2;
}

// Meta data of version edit to table
message VersionEditMeta {
  uint32 space_id = 1;
  uint64 table_id = 2;
  uint64 flushed_sequence = 3;
  repeated AddFileMeta files_to_add = 4;
  repeated DeleteFileMeta files_to_delete = 5;
  uint64 max_file_id = 6;
}

// Meta data of schema update.
message AlterSchemaMeta {
  uint32 space_id = 1;
  uint64 table_id = 2;
  // New schema of the table.
  schema.TableSchema schema = 3;
  // Previous schema version.
  uint32 pre_schema_version = 4;
}

// Meta data of schema update.
message AlterOptionsMeta {
  uint32 space_id = 1;
  uint64 table_id = 2;
  // New options of the table.
  TableOptions options = 3;
}

// Meta update data to persist
message MetaUpdate {
  oneof meta {
    AddTableMeta add_table = 1;
    VersionEditMeta version_edit = 2;
    AlterSchemaMeta alter_schema = 3;
    AlterOptionsMeta alter_options = 4;
    DropTableMeta drop_table = 5;
  }
}

message Snapshot {
  uint64 end_seq = 1;
  AddTableMeta meta = 2;
  VersionEditMeta version_edit = 3;
}

// Options of a table that need to persist
message TableOptions {
  // Segment duration in ms.
  uint64 segment_duration = 1;
  bool enable_ttl = 2;
  uint64 ttl = 3;
  uint32 arena_block_size = 4;
  uint64 num_rows_per_row_group = 5;
  CompactionStrategy compaction_strategy = 6;
  CompactionOptions compaction_options = 7;
  UpdateMode update_mode = 8;
  uint32 write_buffer_size = 9;
  Compression compression = 10;
  // If sampling_segment_duration is true, then the segment duration
  // is still unknown.
  bool sampling_segment_duration = 11;
  StorageFormatHint storage_format_hint = 12;
  LayeredMemtableOptions layered_memtable_options = 13;
}

enum UpdateMode {
  Overwrite = 0;
  Append = 1;
}

message StorageFormatHint {
  oneof hint {
    // Auto means the storage format is automatically determined by CeresDB and
    // its value have no specific meaning.
    int32 auto = 1;
    StorageFormat specific = 2;
  }
}

enum StorageFormat {
  Columnar = 0;
  Hybrid = 1;
}

message CompactionOptions {
  // Options for STCS
  float bucket_low = 1;
  float bucket_high = 2;
  uint32 min_sstable_size = 3;
  uint32 min_threshold = 4;
  uint32 max_threshold = 5;
  // Options for TWCS
  TimeUnit timestamp_resolution = 6;
}

enum TimeUnit {
  NANOSECONDS = 0;
  MICROSECONDS = 1;
  MILLISECONDS = 2;
  SECONDS = 3;
  MINUTES = 4;
  HOURS = 5;
  DAYS = 6;
}

enum CompactionStrategy {
  DEFAULT = 0;
  SIZE_TIERED = 1;
  TIME_WINDOW = 2;
}

enum Compression {
  UNCOMPRESSED = 0;
  LZ4 = 1;
  SNAPPY = 2;
  ZSTD = 3;
}

message LayeredMemtableOptions {
  // Layered memtable will be disable when threshold is 0.
  uint64 mutable_segment_switch_threshold = 1;
}