cpp/src/parquet/printer.cc - arrow - Git at Google

 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
 // regarding copyright ownership.  The ASF licenses this file
 // to you under the Apache License, Version 2.0 (the
 // "License"); you may not use this file except in compliance
 // with the License.  You may obtain a copy of the License at
 //
 //   http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing,
 // software distributed under the License is distributed on an
 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.

 #include "parquet/printer.h"

 #include <cstdint>
 #include <cstdio>
 #include <memory>
 #include <ostream>
 #include <string>
 #include <vector>

 #include "arrow/util/key_value_metadata.h"
 #include "arrow/util/string.h"

 #include "parquet/column_scanner.h"
 #include "parquet/exception.h"
 #include "parquet/file_reader.h"
 #include "parquet/metadata.h"
 #include "parquet/schema.h"
 #include "parquet/statistics.h"
 #include "parquet/types.h"

 namespace parquet {

 class ColumnReader;

 // ----------------------------------------------------------------------
 // ParquetFilePrinter::DebugPrint

 // the fixed initial size is just for an example
 #define COL_WIDTH 30

 void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selected_columns,
                                     bool print_values, bool format_dump,
                                     bool print_key_value_metadata, const char* filename) {
   const FileMetaData* file_metadata = fileReader->metadata().get();

   stream << "File Name: " << filename << "\n";
   stream << "Version: " << ParquetVersionToString(file_metadata->version()) << "\n";
   stream << "Created By: " << file_metadata->created_by() << "\n";
   stream << "Total rows: " << file_metadata->num_rows() << "\n";

   if (print_key_value_metadata && file_metadata->key_value_metadata()) {
     auto key_value_metadata = file_metadata->key_value_metadata();
     int64_t size_of_key_value_metadata = key_value_metadata->size();
     stream << "Key Value File Metadata: " << size_of_key_value_metadata << " entries\n";
     for (int64_t i = 0; i < size_of_key_value_metadata; i++) {
       stream << " Key nr " << i << " " << key_value_metadata->key(i) << ": "
              << key_value_metadata->value(i) << "\n";
     }
   }

   stream << "Number of RowGroups: " << file_metadata->num_row_groups() << "\n";
   stream << "Number of Real Columns: "
          << file_metadata->schema()->group_node()->field_count() << "\n";

   if (selected_columns.size() == 0) {
     for (int i = 0; i < file_metadata->num_columns(); i++) {
       selected_columns.push_back(i);
     }
   } else {
     for (auto i : selected_columns) {
       if (i < 0 || i >= file_metadata->num_columns()) {
         throw ParquetException("Selected column is out of range");
       }
     }
   }

   stream << "Number of Columns: " << file_metadata->num_columns() << "\n";
   stream << "Number of Selected Columns: " << selected_columns.size() << "\n";
   for (auto i : selected_columns) {
     const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
     stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
            << TypeToString(descr->physical_type());
     const auto& logical_type = descr->logical_type();
     if (!logical_type->is_none()) {
       stream << " / " << logical_type->ToString();
     }
     if (descr->converted_type() != ConvertedType::NONE) {
       stream << " / " << ConvertedTypeToString(descr->converted_type());
       if (descr->converted_type() == ConvertedType::DECIMAL) {
         stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
       }
     }
     stream << ")" << std::endl;
   }

   for (int r = 0; r < file_metadata->num_row_groups(); ++r) {
     stream << "--- Row Group: " << r << " ---\n";

     auto group_reader = fileReader->RowGroup(r);
     std::unique_ptr<RowGroupMetaData> group_metadata = file_metadata->RowGroup(r);

     stream << "--- Total Bytes: " << group_metadata->total_byte_size() << " ---\n";
     stream << "--- Total Compressed Bytes: " << group_metadata->total_compressed_size()
            << " ---\n";
     stream << "--- Rows: " << group_metadata->num_rows() << " ---\n";

     // Print column metadata
     for (auto i : selected_columns) {
       auto column_chunk = group_metadata->ColumnChunk(i);
       std::shared_ptr<Statistics> stats = column_chunk->statistics();

       const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
       stream << "Column " << i << std::endl << "  Values: " << column_chunk->num_values();
       if (column_chunk->is_stats_set()) {
         std::string min = stats->EncodeMin(), max = stats->EncodeMax();
         stream << ", Null Values: " << stats->null_count()
                << ", Distinct Values: " << stats->distinct_count() << std::endl
                << "  Max: " << FormatStatValue(descr->physical_type(), max)
                << ", Min: " << FormatStatValue(descr->physical_type(), min);
       } else {
         stream << "  Statistics Not Set";
       }
       stream << std::endl
              << "  Compression: "
              << ::arrow::internal::AsciiToUpper(
                     Codec::GetCodecAsString(column_chunk->compression()))
              << ", Encodings:";
       for (auto encoding : column_chunk->encodings()) {
         stream << " " << EncodingToString(encoding);
       }
       stream << std::endl
              << "  Uncompressed Size: " << column_chunk->total_uncompressed_size()
              << ", Compressed Size: " << column_chunk->total_compressed_size()
              << std::endl;
     }

     if (!print_values) {
       continue;
     }
     stream << "--- Values ---\n";

     static constexpr int bufsize = COL_WIDTH + 1;
     char buffer[bufsize];

     // Create readers for selected columns and print contents
     std::vector<std::shared_ptr<Scanner>> scanners(selected_columns.size(), nullptr);
     int j = 0;
     for (auto i : selected_columns) {
       std::shared_ptr<ColumnReader> col_reader = group_reader->Column(i);
       // This is OK in this method as long as the RowGroupReader does not get
       // deleted
       auto& scanner = scanners[j++] = Scanner::Make(col_reader);

       if (format_dump) {
         stream << "Column " << i << std::endl;
         while (scanner->HasNext()) {
           scanner->PrintNext(stream, 0, true);
           stream << "\n";
         }
         continue;
       }

       snprintf(buffer, bufsize, "%-*s", COL_WIDTH,
                file_metadata->schema()->Column(i)->name().c_str());
       stream << buffer << '|';
     }
     if (format_dump) {
       continue;
     }
     stream << "\n";

     bool hasRow;
     do {
       hasRow = false;
       for (auto scanner : scanners) {
         if (scanner->HasNext()) {
           hasRow = true;
           scanner->PrintNext(stream, COL_WIDTH);
           stream << '|';
         }
       }
       stream << "\n";
     } while (hasRow);
   }
 }

 void ParquetFilePrinter::JSONPrint(std::ostream& stream, std::list<int> selected_columns,
                                    const char* filename) {
   const FileMetaData* file_metadata = fileReader->metadata().get();
   stream << "{\n";
   stream << "  \"FileName\": \"" << filename << "\",\n";
   stream << "  \"Version\": \"" << ParquetVersionToString(file_metadata->version())
          << "\",\n";
   stream << "  \"CreatedBy\": \"" << file_metadata->created_by() << "\",\n";
   stream << "  \"TotalRows\": \"" << file_metadata->num_rows() << "\",\n";
   stream << "  \"NumberOfRowGroups\": \"" << file_metadata->num_row_groups() << "\",\n";
   stream << "  \"NumberOfRealColumns\": \""
          << file_metadata->schema()->group_node()->field_count() << "\",\n";
   stream << "  \"NumberOfColumns\": \"" << file_metadata->num_columns() << "\",\n";

   if (selected_columns.size() == 0) {
     for (int i = 0; i < file_metadata->num_columns(); i++) {
       selected_columns.push_back(i);
     }
   } else {
     for (auto i : selected_columns) {
       if (i < 0 || i >= file_metadata->num_columns()) {
         throw ParquetException("Selected column is out of range");
       }
     }
   }

   stream << "  \"Columns\": [\n";
   int c = 0;
   for (auto i : selected_columns) {
     const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
     stream << "     { \"Id\": \"" << i << "\","
            << " \"Name\": \"" << descr->path()->ToDotString() << "\","
            << " \"PhysicalType\": \"" << TypeToString(descr->physical_type()) << "\","
            << " \"ConvertedType\": \"" << ConvertedTypeToString(descr->converted_type())
            << "\","
            << " \"LogicalType\": " << (descr->logical_type())->ToJSON() << " }";
     c++;
     if (c != static_cast<int>(selected_columns.size())) {
       stream << ",\n";
     }
   }

   stream << "\n  ],\n  \"RowGroups\": [\n";
   for (int r = 0; r < file_metadata->num_row_groups(); ++r) {
     stream << "     {\n       \"Id\": \"" << r << "\", ";

     auto group_reader = fileReader->RowGroup(r);
     std::unique_ptr<RowGroupMetaData> group_metadata = file_metadata->RowGroup(r);

     stream << " \"TotalBytes\": \"" << group_metadata->total_byte_size() << "\", ";
     stream << " \"TotalCompressedBytes\": \"" << group_metadata->total_compressed_size()
            << "\", ";
     stream << " \"Rows\": \"" << group_metadata->num_rows() << "\",\n";

     // Print column metadata
     stream << "       \"ColumnChunks\": [\n";
     int c1 = 0;
     for (auto i : selected_columns) {
       auto column_chunk = group_metadata->ColumnChunk(i);
       std::shared_ptr<Statistics> stats = column_chunk->statistics();

       const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
       stream << "          {\"Id\": \"" << i << "\", \"Values\": \""
              << column_chunk->num_values() << "\", "
              << "\"StatsSet\": ";
       if (column_chunk->is_stats_set()) {
         stream << "\"True\", \"Stats\": {";
         std::string min = stats->EncodeMin(), max = stats->EncodeMax();
         stream << "\"NumNulls\": \"" << stats->null_count() << "\", "
                << "\"DistinctValues\": \"" << stats->distinct_count() << "\", "
                << "\"Max\": \"" << FormatStatValue(descr->physical_type(), max) << "\", "
                << "\"Min\": \"" << FormatStatValue(descr->physical_type(), min)
                << "\" },";
       } else {
         stream << "\"False\",";
       }
       stream << "\n           \"Compression\": \""
              << ::arrow::internal::AsciiToUpper(
                     Codec::GetCodecAsString(column_chunk->compression()))
              << "\", \"Encodings\": \"";
       for (auto encoding : column_chunk->encodings()) {
         stream << EncodingToString(encoding) << " ";
       }
       stream << "\", "
              << "\"UncompressedSize\": \"" << column_chunk->total_uncompressed_size()
              << "\", \"CompressedSize\": \"" << column_chunk->total_compressed_size();

       // end of a ColumnChunk
       stream << "\" }";
       c1++;
       if (c1 != static_cast<int>(selected_columns.size())) {
         stream << ",\n";
       }
     }

     stream << "\n        ]\n     }";
     if ((r + 1) != static_cast<int>(file_metadata->num_row_groups())) {
       stream << ",\n";
     }
   }
   stream << "\n  ]\n}\n";
 }

 }  // namespace parquet
	// Licensed to the Apache Software Foundation (ASF) under one
	// or more contributor license agreements. See the NOTICE file
	// distributed with this work for additional information
	// regarding copyright ownership. The ASF licenses this file
	// to you under the Apache License, Version 2.0 (the
	// "License"); you may not use this file except in compliance
	// with the License. You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing,
	// software distributed under the License is distributed on an
	// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	// KIND, either express or implied. See the License for the
	// specific language governing permissions and limitations
	// under the License.

	#include "parquet/printer.h"

	#include <cstdint>
	#include <cstdio>
	#include <memory>
	#include <ostream>
	#include <string>
	#include <vector>

	#include "arrow/util/key_value_metadata.h"
	#include "arrow/util/string.h"

	#include "parquet/column_scanner.h"
	#include "parquet/exception.h"
	#include "parquet/file_reader.h"
	#include "parquet/metadata.h"
	#include "parquet/schema.h"
	#include "parquet/statistics.h"
	#include "parquet/types.h"

	namespace parquet {

	class ColumnReader;

	// ----------------------------------------------------------------------
	// ParquetFilePrinter::DebugPrint

	// the fixed initial size is just for an example
	#define COL_WIDTH 30

	void ParquetFilePrinter::DebugPrint(std::ostream& stream, std::list<int> selected_columns,
	bool print_values, bool format_dump,
	bool print_key_value_metadata, const char* filename) {
	const FileMetaData* file_metadata = fileReader->metadata().get();

	stream << "File Name: " << filename << "\n";
	stream << "Version: " << ParquetVersionToString(file_metadata->version()) << "\n";
	stream << "Created By: " << file_metadata->created_by() << "\n";
	stream << "Total rows: " << file_metadata->num_rows() << "\n";

	if (print_key_value_metadata && file_metadata->key_value_metadata()) {
	auto key_value_metadata = file_metadata->key_value_metadata();
	int64_t size_of_key_value_metadata = key_value_metadata->size();
	stream << "Key Value File Metadata: " << size_of_key_value_metadata << " entries\n";
	for (int64_t i = 0; i < size_of_key_value_metadata; i++) {
	stream << " Key nr " << i << " " << key_value_metadata->key(i) << ": "
	<< key_value_metadata->value(i) << "\n";
	}
	}

	stream << "Number of RowGroups: " << file_metadata->num_row_groups() << "\n";
	stream << "Number of Real Columns: "
	<< file_metadata->schema()->group_node()->field_count() << "\n";

	if (selected_columns.size() == 0) {
	for (int i = 0; i < file_metadata->num_columns(); i++) {
	selected_columns.push_back(i);
	}
	} else {
	for (auto i : selected_columns) {
	if (i < 0 \|\| i >= file_metadata->num_columns()) {
	throw ParquetException("Selected column is out of range");
	}
	}
	}

	stream << "Number of Columns: " << file_metadata->num_columns() << "\n";
	stream << "Number of Selected Columns: " << selected_columns.size() << "\n";
	for (auto i : selected_columns) {
	const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
	stream << "Column " << i << ": " << descr->path()->ToDotString() << " ("
	<< TypeToString(descr->physical_type());
	const auto& logical_type = descr->logical_type();
	if (!logical_type->is_none()) {
	stream << " / " << logical_type->ToString();
	}
	if (descr->converted_type() != ConvertedType::NONE) {
	stream << " / " << ConvertedTypeToString(descr->converted_type());
	if (descr->converted_type() == ConvertedType::DECIMAL) {
	stream << "(" << descr->type_precision() << "," << descr->type_scale() << ")";
	}
	}
	stream << ")" << std::endl;
	}

	for (int r = 0; r < file_metadata->num_row_groups(); ++r) {
	stream << "--- Row Group: " << r << " ---\n";

	auto group_reader = fileReader->RowGroup(r);
	std::unique_ptr<RowGroupMetaData> group_metadata = file_metadata->RowGroup(r);

	stream << "--- Total Bytes: " << group_metadata->total_byte_size() << " ---\n";
	stream << "--- Total Compressed Bytes: " << group_metadata->total_compressed_size()
	<< " ---\n";
	stream << "--- Rows: " << group_metadata->num_rows() << " ---\n";

	// Print column metadata
	for (auto i : selected_columns) {
	auto column_chunk = group_metadata->ColumnChunk(i);
	std::shared_ptr<Statistics> stats = column_chunk->statistics();

	const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
	stream << "Column " << i << std::endl << " Values: " << column_chunk->num_values();
	if (column_chunk->is_stats_set()) {
	std::string min = stats->EncodeMin(), max = stats->EncodeMax();
	stream << ", Null Values: " << stats->null_count()
	<< ", Distinct Values: " << stats->distinct_count() << std::endl
	<< " Max: " << FormatStatValue(descr->physical_type(), max)
	<< ", Min: " << FormatStatValue(descr->physical_type(), min);
	} else {
	stream << " Statistics Not Set";
	}
	stream << std::endl
	<< " Compression: "
	<< ::arrow::internal::AsciiToUpper(
	Codec::GetCodecAsString(column_chunk->compression()))
	<< ", Encodings:";
	for (auto encoding : column_chunk->encodings()) {
	stream << " " << EncodingToString(encoding);
	}
	stream << std::endl
	<< " Uncompressed Size: " << column_chunk->total_uncompressed_size()
	<< ", Compressed Size: " << column_chunk->total_compressed_size()
	<< std::endl;
	}

	if (!print_values) {
	continue;
	}
	stream << "--- Values ---\n";

	static constexpr int bufsize = COL_WIDTH + 1;
	char buffer[bufsize];

	// Create readers for selected columns and print contents
	std::vector<std::shared_ptr<Scanner>> scanners(selected_columns.size(), nullptr);
	int j = 0;
	for (auto i : selected_columns) {
	std::shared_ptr<ColumnReader> col_reader = group_reader->Column(i);
	// This is OK in this method as long as the RowGroupReader does not get
	// deleted
	auto& scanner = scanners[j++] = Scanner::Make(col_reader);

	if (format_dump) {
	stream << "Column " << i << std::endl;
	while (scanner->HasNext()) {
	scanner->PrintNext(stream, 0, true);
	stream << "\n";
	}
	continue;
	}

	snprintf(buffer, bufsize, "%-*s", COL_WIDTH,
	file_metadata->schema()->Column(i)->name().c_str());
	stream << buffer << '\|';
	}
	if (format_dump) {
	continue;
	}
	stream << "\n";

	bool hasRow;
	do {
	hasRow = false;
	for (auto scanner : scanners) {
	if (scanner->HasNext()) {
	hasRow = true;
	scanner->PrintNext(stream, COL_WIDTH);
	stream << '\|';
	}
	}
	stream << "\n";
	} while (hasRow);
	}
	}

	void ParquetFilePrinter::JSONPrint(std::ostream& stream, std::list<int> selected_columns,
	const char* filename) {
	const FileMetaData* file_metadata = fileReader->metadata().get();
	stream << "{\n";
	stream << " \"FileName\": \"" << filename << "\",\n";
	stream << " \"Version\": \"" << ParquetVersionToString(file_metadata->version())
	<< "\",\n";
	stream << " \"CreatedBy\": \"" << file_metadata->created_by() << "\",\n";
	stream << " \"TotalRows\": \"" << file_metadata->num_rows() << "\",\n";
	stream << " \"NumberOfRowGroups\": \"" << file_metadata->num_row_groups() << "\",\n";
	stream << " \"NumberOfRealColumns\": \""
	<< file_metadata->schema()->group_node()->field_count() << "\",\n";
	stream << " \"NumberOfColumns\": \"" << file_metadata->num_columns() << "\",\n";

	if (selected_columns.size() == 0) {
	for (int i = 0; i < file_metadata->num_columns(); i++) {
	selected_columns.push_back(i);
	}
	} else {
	for (auto i : selected_columns) {
	if (i < 0 \|\| i >= file_metadata->num_columns()) {
	throw ParquetException("Selected column is out of range");
	}
	}
	}

	stream << " \"Columns\": [\n";
	int c = 0;
	for (auto i : selected_columns) {
	const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
	stream << " { \"Id\": \"" << i << "\","
	<< " \"Name\": \"" << descr->path()->ToDotString() << "\","
	<< " \"PhysicalType\": \"" << TypeToString(descr->physical_type()) << "\","
	<< " \"ConvertedType\": \"" << ConvertedTypeToString(descr->converted_type())
	<< "\","
	<< " \"LogicalType\": " << (descr->logical_type())->ToJSON() << " }";
	c++;
	if (c != static_cast<int>(selected_columns.size())) {
	stream << ",\n";
	}
	}

	stream << "\n ],\n \"RowGroups\": [\n";
	for (int r = 0; r < file_metadata->num_row_groups(); ++r) {
	stream << " {\n \"Id\": \"" << r << "\", ";

	auto group_reader = fileReader->RowGroup(r);
	std::unique_ptr<RowGroupMetaData> group_metadata = file_metadata->RowGroup(r);

	stream << " \"TotalBytes\": \"" << group_metadata->total_byte_size() << "\", ";
	stream << " \"TotalCompressedBytes\": \"" << group_metadata->total_compressed_size()
	<< "\", ";
	stream << " \"Rows\": \"" << group_metadata->num_rows() << "\",\n";

	// Print column metadata
	stream << " \"ColumnChunks\": [\n";
	int c1 = 0;
	for (auto i : selected_columns) {
	auto column_chunk = group_metadata->ColumnChunk(i);
	std::shared_ptr<Statistics> stats = column_chunk->statistics();

	const ColumnDescriptor* descr = file_metadata->schema()->Column(i);
	stream << " {\"Id\": \"" << i << "\", \"Values\": \""
	<< column_chunk->num_values() << "\", "
	<< "\"StatsSet\": ";
	if (column_chunk->is_stats_set()) {
	stream << "\"True\", \"Stats\": {";
	std::string min = stats->EncodeMin(), max = stats->EncodeMax();
	stream << "\"NumNulls\": \"" << stats->null_count() << "\", "
	<< "\"DistinctValues\": \"" << stats->distinct_count() << "\", "
	<< "\"Max\": \"" << FormatStatValue(descr->physical_type(), max) << "\", "
	<< "\"Min\": \"" << FormatStatValue(descr->physical_type(), min)
	<< "\" },";
	} else {
	stream << "\"False\",";
	}
	stream << "\n \"Compression\": \""
	<< ::arrow::internal::AsciiToUpper(
	Codec::GetCodecAsString(column_chunk->compression()))
	<< "\", \"Encodings\": \"";
	for (auto encoding : column_chunk->encodings()) {
	stream << EncodingToString(encoding) << " ";
	}
	stream << "\", "
	<< "\"UncompressedSize\": \"" << column_chunk->total_uncompressed_size()
	<< "\", \"CompressedSize\": \"" << column_chunk->total_compressed_size();

	// end of a ColumnChunk
	stream << "\" }";
	c1++;
	if (c1 != static_cast<int>(selected_columns.size())) {
	stream << ",\n";
	}
	}

	stream << "\n ]\n }";
	if ((r + 1) != static_cast<int>(file_metadata->num_row_groups())) {
	stream << ",\n";
	}
	}
	stream << "\n ]\n}\n";
	}

	} // namespace parquet