| // Licensed to the Apache Software Foundation (ASF) under one |
| // or more contributor license agreements. See the NOTICE file |
| // distributed with this work for additional information |
| // regarding copyright ownership. The ASF licenses this file |
| // to you under the Apache License, Version 2.0 (the |
| // "License"); you may not use this file except in compliance |
| // with the License. You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, |
| // software distributed under the License is distributed on an |
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| // KIND, either express or implied. See the License for the |
| // specific language governing permissions and limitations |
| // under the License. |
| syntax = "proto2"; |
| package kudu.cfile; |
| |
| option java_package = "org.apache.kudu.cfile"; |
| |
| import "kudu/common/common.proto"; |
| import "kudu/util/compression/compression.proto"; |
| import "kudu/util/pb_util.proto"; |
| |
| message FileMetadataPairPB { |
| required string key = 1; |
| |
| // We redact the metadata values here because, despite being "metadata" |
| // from the perspective of the CFile, they can in many cases store actual |
| // row data (e.g min/max key values). |
| required bytes value = 2 [ (REDACT) = true ]; |
| } |
| |
| message CFileHeaderPB { |
| // These fields were used in Kudu <= 1.2, for the original header cfile format |
| // corresponding to the 'kuducfil' magic string. |
| // The new format ('kuducfl2' magic) no longer uses them, instead using the |
| // "feature flags" in the CFileFooterPB. |
| // required int32 major_version = 1; |
| // required int32 minor_version = 2; |
| repeated FileMetadataPairPB metadata = 3; |
| } |
| |
| message BlockPointerPB { |
| required int64 offset = 1; |
| required int32 size = 2; |
| } |
| |
| message BTreeInfoPB { |
| required BlockPointerPB root_block = 1; |
| } |
| |
| message IndexBlockTrailerPB { |
| required int32 num_entries = 1; |
| |
| enum BlockType { |
| UNKNOWN = 999; |
| LEAF = 0; |
| INTERNAL = 1; |
| }; |
| required BlockType type = 2; |
| } |
| // TODO: name all the PBs with *PB convention |
| |
| message CFileFooterPB { |
| required kudu.DataType data_type = 1; |
| required EncodingType encoding = 2; |
| |
| // Total number of values in the file. |
| required int64 num_values = 3; |
| |
| optional BTreeInfoPB posidx_info = 4; |
| optional BTreeInfoPB validx_info = 5; |
| |
| optional CompressionType compression = 6 [default=NO_COMPRESSION]; |
| |
| repeated FileMetadataPairPB metadata = 7; |
| |
| optional bool is_type_nullable = 8 [default=false]; |
| |
| // Block pointer for dictionary block if the cfile is dictionary encoded. |
| // Only for dictionary encoding. |
| optional BlockPointerPB dict_block_ptr = 9; |
| |
| // Bitsets of features used by this cfile. |
| // |
| // We use a uint32 bitset instead of a repeated enum in order to save memory since |
| // these footers are kept around in memory for every open CFile. If we ever run out |
| // of bits, we can use the last bit of either list to enable a second bitset. |
| // |
| // We designate each feature as either compatible or incompatible. |
| // |
| // INCOMPATIBLE |
| // --------------- |
| // An incompatible feature is one which must be supported by a reader |
| // in order to properly read the file. If a reader sees an unknown bit set in this list, |
| // it should refuse to read the file. |
| // |
| // Example: an incompatible change to the format of one of the block encodings. |
| // If a reader isn't aware of the flag, it would read incorrect data. |
| // |
| // COMPATIBLE |
| // ----------- |
| // A compatible feature is one which may be safely ignored by a reader without |
| // incorrect results. A reader may proceed but may not be taking advantage of |
| // all available index structures, etc. |
| // |
| // An example case here would be something like an additional index structure |
| // or summary statistic which an aware reader could potentially use, but an |
| // old reader could safely ignore. |
| optional uint32 incompatible_features = 10; |
| optional uint32 compatible_features = 11; |
| } |
| |
| |
| message BloomBlockHeaderPB { |
| required int32 num_hash_functions = 1; |
| } |