blob: 7002fb868c32fe361d7c3cf9e12f51b90e42521f [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Logic handling reading from Avro format at user level.
mod block;
pub mod datum;
pub mod single_object;
use std::{collections::HashMap, io::Read, marker::PhantomData};
use block::Block;
use bon::bon;
use serde::de::DeserializeOwned;
use crate::{AvroResult, schema::Schema, types::Value, util::is_human_readable};
/// Main interface for reading Avro formatted values.
///
/// To be used as an iterator:
///
/// ```no_run
/// # use apache_avro::Reader;
/// # use std::io::Cursor;
/// # let input = Cursor::new(Vec::<u8>::new());
/// for value in Reader::new(input).unwrap() {
/// match value {
/// Ok(v) => println!("{:?}", v),
/// Err(e) => println!("Error: {}", e),
/// };
/// }
/// ```
pub struct Reader<'a, R> {
block: Block<'a, R>,
reader_schema: Option<&'a Schema>,
errored: bool,
should_resolve_schema: bool,
}
#[bon]
impl<'a, R: Read> Reader<'a, R> {
/// Creates a `Reader` given something implementing the `io::Read` trait to read from.
/// No reader `Schema` will be set.
///
/// **NOTE** The avro header is going to be read automatically upon creation of the `Reader`.
pub fn new(reader: R) -> AvroResult<Reader<'a, R>> {
Reader::builder(reader).build()
}
/// Creates a `Reader` given something implementing the `io::Read` trait to read from.
/// With an optional reader `Schema` and optional schemata to use for resolving schema
/// references.
///
/// **NOTE** The avro header is going to be read automatically upon creation of the `Reader`.
#[builder(finish_fn = build)]
pub fn builder(
#[builder(start_fn)] reader: R,
/// The schema the written data needs to be adapted to.
///
/// This is currently not compatible with [`ReaderDeser`] and will panic during reading if the
/// writer schema is not the same as the reader schema.
reader_schema: Option<&'a Schema>,
schemata: Option<Vec<&'a Schema>>,
#[builder(default = is_human_readable())] human_readable: bool,
) -> AvroResult<Reader<'a, R>> {
let schemata =
schemata.unwrap_or_else(|| reader_schema.map(|rs| vec![rs]).unwrap_or_default());
let block = Block::new(reader, schemata, human_readable)?;
let mut reader = Reader {
block,
reader_schema,
errored: false,
should_resolve_schema: false,
};
// Check if the reader and writer schemas disagree.
reader.should_resolve_schema =
reader_schema.is_some_and(|reader_schema| reader.writer_schema() != reader_schema);
Ok(reader)
}
/// Get a reference to the writer `Schema`.
#[inline]
pub fn writer_schema(&self) -> &Schema {
&self.block.writer_schema
}
/// Get a reference to the optional reader `Schema`.
#[inline]
pub fn reader_schema(&self) -> Option<&Schema> {
self.reader_schema
}
/// Get a reference to the user metadata.
#[inline]
pub fn user_metadata(&self) -> &HashMap<String, Vec<u8>> {
&self.block.user_metadata
}
/// Convert this reader into an iterator that deserializes to `T`.
pub fn into_deser_iter<T: DeserializeOwned>(self) -> ReaderDeser<'a, R, T> {
ReaderDeser {
inner: self,
phantom: PhantomData,
}
}
#[inline]
fn read_next(&mut self) -> AvroResult<Option<Value>> {
let read_schema = if self.should_resolve_schema {
self.reader_schema
} else {
None
};
self.block.read_next(read_schema)
}
fn read_next_deser<T: DeserializeOwned>(&mut self) -> AvroResult<Option<T>> {
// TODO: Implement SchemaAwareResolvingDeserializer
assert!(
!self.should_resolve_schema,
"Schema aware deserialisation does not resolve schemas yet"
);
self.block.read_next_deser(self.reader_schema)
}
}
impl<R: Read> Iterator for Reader<'_, R> {
type Item = AvroResult<Value>;
fn next(&mut self) -> Option<Self::Item> {
// to prevent keep on reading after the first error occurs
if self.errored {
return None;
};
match self.read_next() {
Ok(opt) => opt.map(Ok),
Err(e) => {
self.errored = true;
Some(Err(e))
}
}
}
}
/// Wrapper around [`Reader`] where the iterator deserializes `T`.
pub struct ReaderDeser<'a, R, T> {
inner: Reader<'a, R>,
phantom: PhantomData<T>,
}
impl<R: Read, T: DeserializeOwned> Iterator for ReaderDeser<'_, R, T> {
type Item = AvroResult<T>;
fn next(&mut self) -> Option<Self::Item> {
// Don't continue when we've errored before
if self.inner.errored {
return None;
}
match self.inner.read_next_deser::<T>() {
Ok(opt) => opt.map(Ok),
Err(e) => {
self.inner.errored = true;
Some(Err(e))
}
}
}
}
/// Reads the marker bytes from Avro bytes generated earlier by a `Writer`
///
/// # Panics
/// Will panic if `bytes` does not contain at least 16 bytes.
pub fn read_marker(bytes: &[u8]) -> [u8; 16] {
assert!(
bytes.len() > 16,
"The bytes are too short to read a marker from them"
);
let mut marker = [0_u8; 16];
marker.clone_from_slice(&bytes[(bytes.len() - 16)..]);
marker
}
#[cfg(test)]
mod tests {
use std::io::Cursor;
use apache_avro_test_helper::TestResult;
use pretty_assertions::assert_eq;
use super::*;
use crate::types::Record;
const SCHEMA: &str = r#"
{
"type": "record",
"name": "test",
"fields": [
{
"name": "a",
"type": "long",
"default": 42
},
{
"name": "b",
"type": "string"
}
]
}
"#;
const ENCODED: &[u8] = &[
79u8, 98u8, 106u8, 1u8, 4u8, 22u8, 97u8, 118u8, 114u8, 111u8, 46u8, 115u8, 99u8, 104u8,
101u8, 109u8, 97u8, 222u8, 1u8, 123u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, 34u8,
114u8, 101u8, 99u8, 111u8, 114u8, 100u8, 34u8, 44u8, 34u8, 110u8, 97u8, 109u8, 101u8, 34u8,
58u8, 34u8, 116u8, 101u8, 115u8, 116u8, 34u8, 44u8, 34u8, 102u8, 105u8, 101u8, 108u8,
100u8, 115u8, 34u8, 58u8, 91u8, 123u8, 34u8, 110u8, 97u8, 109u8, 101u8, 34u8, 58u8, 34u8,
97u8, 34u8, 44u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, 34u8, 108u8, 111u8, 110u8,
103u8, 34u8, 44u8, 34u8, 100u8, 101u8, 102u8, 97u8, 117u8, 108u8, 116u8, 34u8, 58u8, 52u8,
50u8, 125u8, 44u8, 123u8, 34u8, 110u8, 97u8, 109u8, 101u8, 34u8, 58u8, 34u8, 98u8, 34u8,
44u8, 34u8, 116u8, 121u8, 112u8, 101u8, 34u8, 58u8, 34u8, 115u8, 116u8, 114u8, 105u8,
110u8, 103u8, 34u8, 125u8, 93u8, 125u8, 20u8, 97u8, 118u8, 114u8, 111u8, 46u8, 99u8, 111u8,
100u8, 101u8, 99u8, 8u8, 110u8, 117u8, 108u8, 108u8, 0u8, 94u8, 61u8, 54u8, 221u8, 190u8,
207u8, 108u8, 180u8, 158u8, 57u8, 114u8, 40u8, 173u8, 199u8, 228u8, 239u8, 4u8, 20u8, 54u8,
6u8, 102u8, 111u8, 111u8, 84u8, 6u8, 98u8, 97u8, 114u8, 94u8, 61u8, 54u8, 221u8, 190u8,
207u8, 108u8, 180u8, 158u8, 57u8, 114u8, 40u8, 173u8, 199u8, 228u8, 239u8,
];
#[test]
fn test_reader_iterator() -> TestResult {
let schema = Schema::parse_str(SCHEMA)?;
let reader = Reader::builder(ENCODED).reader_schema(&schema).build()?;
let mut record1 = Record::new(&schema).unwrap();
record1.put("a", 27i64);
record1.put("b", "foo");
let mut record2 = Record::new(&schema).unwrap();
record2.put("a", 42i64);
record2.put("b", "bar");
let expected = [record1.into(), record2.into()];
for (i, value) in reader.enumerate() {
assert_eq!(value?, expected[i]);
}
Ok(())
}
#[test]
fn test_reader_invalid_header() -> TestResult {
let schema = Schema::parse_str(SCHEMA)?;
let mut invalid = &ENCODED[1..];
assert!(
Reader::builder(&mut invalid)
.reader_schema(&schema)
.build()
.is_err()
);
Ok(())
}
#[test]
fn test_reader_invalid_block() -> TestResult {
let schema = Schema::parse_str(SCHEMA)?;
let mut invalid = &ENCODED[0..ENCODED.len() - 19];
let reader = Reader::builder(&mut invalid)
.reader_schema(&schema)
.build()?;
for value in reader {
assert!(value.is_err());
}
Ok(())
}
#[test]
fn test_reader_empty_buffer() -> TestResult {
let empty = Cursor::new(Vec::new());
assert!(Reader::new(empty).is_err());
Ok(())
}
#[test]
fn test_reader_only_header() -> TestResult {
let mut invalid = &ENCODED[..165];
let reader = Reader::new(&mut invalid)?;
for value in reader {
assert!(value.is_err());
}
Ok(())
}
#[test]
fn test_avro_3405_read_user_metadata_success() -> TestResult {
use crate::writer::Writer;
let schema = Schema::parse_str(SCHEMA)?;
let mut writer = Writer::new(&schema, Vec::new())?;
let mut user_meta_data: HashMap<String, Vec<u8>> = HashMap::new();
user_meta_data.insert(
"stringKey".to_string(),
"stringValue".to_string().into_bytes(),
);
user_meta_data.insert("bytesKey".to_string(), b"bytesValue".to_vec());
user_meta_data.insert("vecKey".to_string(), vec![1, 2, 3]);
for (k, v) in user_meta_data.iter() {
writer.add_user_metadata(k.to_string(), v)?;
}
let mut record = Record::new(&schema).unwrap();
record.put("a", 27i64);
record.put("b", "foo");
writer.append_value(record.clone())?;
writer.append_value(record.clone())?;
writer.flush()?;
let result = writer.into_inner()?;
let reader = Reader::new(&result[..])?;
assert_eq!(reader.user_metadata(), &user_meta_data);
Ok(())
}
#[cfg(not(feature = "snappy"))]
#[test]
fn test_avro_3549_read_not_enabled_codec() {
let snappy_compressed_avro = vec![
79, 98, 106, 1, 4, 22, 97, 118, 114, 111, 46, 115, 99, 104, 101, 109, 97, 210, 1, 123,
34, 102, 105, 101, 108, 100, 115, 34, 58, 91, 123, 34, 110, 97, 109, 101, 34, 58, 34,
110, 117, 109, 34, 44, 34, 116, 121, 112, 101, 34, 58, 34, 115, 116, 114, 105, 110,
103, 34, 125, 93, 44, 34, 110, 97, 109, 101, 34, 58, 34, 101, 118, 101, 110, 116, 34,
44, 34, 110, 97, 109, 101, 115, 112, 97, 99, 101, 34, 58, 34, 101, 120, 97, 109, 112,
108, 101, 110, 97, 109, 101, 115, 112, 97, 99, 101, 34, 44, 34, 116, 121, 112, 101, 34,
58, 34, 114, 101, 99, 111, 114, 100, 34, 125, 20, 97, 118, 114, 111, 46, 99, 111, 100,
101, 99, 12, 115, 110, 97, 112, 112, 121, 0, 213, 209, 241, 208, 200, 110, 164, 47,
203, 25, 90, 235, 161, 167, 195, 177, 2, 20, 4, 12, 6, 49, 50, 51, 115, 38, 58, 0, 213,
209, 241, 208, 200, 110, 164, 47, 203, 25, 90, 235, 161, 167, 195, 177,
];
if let Err(err) = Reader::new(snappy_compressed_avro.as_slice()) {
assert_eq!("Codec 'snappy' is not supported/enabled", err.to_string());
} else {
panic!("Expected an error in the reading of the codec!");
}
}
}