src/kudu/cfile/cfile.proto

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
syntax = "proto2";
package kudu.cfile;

option java_package = "org.apache.kudu.cfile";

import "kudu/common/common.proto";
import "kudu/util/compression/compression.proto";
import "kudu/util/pb_util.proto";

message FileMetadataPairPB {
  required string key = 1;

  // We redact the metadata values here because, despite being "metadata"
  // from the perspective of the CFile, they can in many cases store actual
  // row data (e.g min/max key values).
  required bytes value = 2 [ (REDACT) = true ];
}

message CFileHeaderPB {
  // These fields were used in Kudu <= 1.2, for the original header cfile format
  // corresponding to the 'kuducfil' magic string.
  // The new format ('kuducfl2' magic) no longer uses them, instead using the
  // "feature flags" in the CFileFooterPB.
  // required int32 major_version = 1;
  // required int32 minor_version = 2;
  repeated FileMetadataPairPB metadata = 3;
}

message BlockPointerPB {
  required int64 offset = 1;
  required int32 size = 2;
}

message BTreeInfoPB {
  required BlockPointerPB root_block = 1;
}

message IndexBlockTrailerPB {
  required int32 num_entries = 1;

  enum BlockType {
    UNKNOWN = 999;
    LEAF = 0;
    INTERNAL = 1;
  };
  required BlockType type = 2;
}
// TODO: name all the PBs with *PB convention

message CFileFooterPB {
  required kudu.DataType data_type = 1;
  required EncodingType encoding = 2;

  // Total number of values in the file.
  required int64 num_values = 3;

  optional BTreeInfoPB posidx_info = 4;
  optional BTreeInfoPB validx_info = 5;

  optional CompressionType compression = 6 [default=NO_COMPRESSION];

  repeated FileMetadataPairPB metadata = 7;

  optional bool is_type_nullable = 8 [default=false];

  // Block pointer for dictionary block if the cfile is dictionary encoded.
  // Only for dictionary encoding.
  optional BlockPointerPB dict_block_ptr = 9;

  // Bitsets of features used by this cfile.
  //
  // We use a uint32 bitset instead of a repeated enum in order to save memory since
  // these footers are kept around in memory for every open CFile. If we ever run out
  // of bits, we can use the last bit of either list to enable a second bitset.
  //
  // We designate each feature as either compatible or incompatible.
  //
  // INCOMPATIBLE
  // ---------------
  // An incompatible feature is one which must be supported by a reader
  // in order to properly read the file. If a reader sees an unknown bit set in this list,
  // it should refuse to read the file.
  //
  // Example: an incompatible change to the format of one of the block encodings.
  // If a reader isn't aware of the flag, it would read incorrect data.
  //
  // COMPATIBLE
  // -----------
  // A compatible feature is one which may be safely ignored by a reader without
  // incorrect results. A reader may proceed but may not be taking advantage of
  // all available index structures, etc.
  //
  // An example case here would be something like an additional index structure
  // or summary statistic which an aware reader could potentially use, but an
  // old reader could safely ignore.
  optional uint32 incompatible_features = 10;
  optional uint32 compatible_features = 11;
}


message BloomBlockHeaderPB {
  required int32 num_hash_functions = 1;
}