| <!-- |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, |
| software distributed under the License is distributed on an |
| "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| KIND, either express or implied. See the License for the |
| specific language governing permissions and limitations |
| under the License. |
| --> |
| |
| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <title>Rust API - Mosaic</title> |
| <link rel="stylesheet" href="css/style.css"> |
| <script src="js/main.js"></script> |
| </head> |
| <body> |
| <button class="menu-toggle" aria-label="Menu">☰</button> |
| <div class="overlay"></div> |
| |
| <aside class="sidebar"> |
| <div class="sidebar-header"> |
| <h2>Mosaic</h2> |
| <p>Columnar-bucket hybrid format</p> |
| </div> |
| <nav> |
| <ul> |
| <li><a href="index.html">Home</a></li> |
| <li><a href="design.html">Design</a></li> |
| <li><a href="rust-api.html" class="active">Rust API</a></li> |
| <li><a href="java-api.html">Java API</a></li> |
| <li><a href="python-api.html">Python API</a></li> |
| <li><a href="cpp-api.html">C++ API</a></li> |
| </ul> |
| </nav> |
| <div class="sidebar-footer"> |
| <button class="theme-toggle">Dark Mode</button> |
| </div> |
| </aside> |
| |
| <main class="main"> |
| <div class="content"> |
| <h1>Rust API</h1> |
| <p class="subtitle">Build from source and write your first Mosaic file in Rust.</p> |
| |
| <h2>Building from Source</h2> |
| <pre><code>git clone <repo-url> |
| cd mosaic |
| cargo build</code></pre> |
| |
| <p>Run the test suite to verify everything works:</p> |
| <pre><code>cargo test</code></pre> |
| |
| <h2>Rust: Writing a File</h2> |
| <p> |
| Add <code>mosaic-core</code> as a dependency (path dependency within the workspace), |
| then use <code>MosaicWriter</code> to create a file. Data is written as Arrow |
| <code>RecordBatch</code> objects: |
| </p> |
| <pre><code><span class="kw">use</span> std::sync::Arc; |
| <span class="kw">use</span> arrow::datatypes::{DataType, Field, Schema}; |
| <span class="kw">use</span> arrow::array::*; |
| <span class="kw">use</span> arrow::record_batch::RecordBatch; |
| <span class="kw">use</span> mosaic_core::writer::{MosaicWriter, WriterOptions}; |
| <span class="kw">use</span> mosaic_core::spec::*; |
| |
| <span class="cmt">// 1. Define an Arrow Schema</span> |
| <span class="kw">let</span> arrow_schema = Schema::new(<span class="kw">vec!</span>[ |
| Field::new(<span class="str">"age"</span>, DataType::Int32, <span class="kw">true</span>), |
| Field::new(<span class="str">"name"</span>, DataType::Utf8, <span class="kw">true</span>), |
| Field::new(<span class="str">"score"</span>, DataType::Float64, <span class="kw">true</span>), |
| ]); |
| |
| <span class="cmt">// 2. Create writer (writes to any OutputFile implementation)</span> |
| <span class="kw">let</span> <span class="kw">mut</span> writer = MosaicWriter::new(output, &arrow_schema, WriterOptions { |
| num_buckets: <span class="num">2</span>, |
| compression: COMPRESSION_ZSTD, |
| ..Default::default() |
| })?; |
| |
| <span class="cmt">// 3. Build an Arrow RecordBatch and write it</span> |
| <span class="kw">let</span> ages = Int32Array::from((<span class="num">0</span>..<span class="num">1000</span>).map(|i| <span class="num">20</span> + (i % <span class="num">50</span>)).collect::<Vec<_>>()); |
| <span class="kw">let</span> names = StringArray::from((<span class="num">0</span>..<span class="num">1000</span>).map(|i| <span class="kw">format!</span>(<span class="str">"user_{}"</span>, i)).collect::<Vec<_>>()); |
| <span class="kw">let</span> scores = Float64Array::from((<span class="num">0</span>..<span class="num">1000</span>).map(|i| i <span class="kw">as</span> <span class="ty">f64</span> * <span class="num">1.5</span>).collect::<Vec<_>>()); |
| <span class="kw">let</span> batch = RecordBatch::try_new( |
| Arc::new(Schema::new(<span class="kw">vec!</span>[ |
| Field::new(<span class="str">"age"</span>, DataType::Int32, <span class="kw">true</span>), |
| Field::new(<span class="str">"name"</span>, DataType::Utf8, <span class="kw">true</span>), |
| Field::new(<span class="str">"score"</span>, DataType::Float64, <span class="kw">true</span>), |
| ])), |
| <span class="kw">vec!</span>[Arc::new(ages), Arc::new(names), Arc::new(scores)], |
| ).unwrap(); |
| writer.write_batch(&batch).unwrap(); |
| |
| <span class="cmt">// 4. Finalize</span> |
| writer.close().unwrap(); |
| |
| <span class="cmt">// Estimated file size (for file rolling decisions)</span> |
| <span class="kw">let</span> est = writer.estimated_file_size();</code></pre> |
| |
| <h3>Writer Methods</h3> |
| <table> |
| <thead> |
| <tr><th>Method</th><th>Return</th><th>Description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>write_batch(&RecordBatch)</code></td><td><code>io::Result<()></code></td><td>Write an Arrow RecordBatch</td></tr> |
| <tr><td><code>estimated_file_size()</code></td><td><code>u64</code></td><td>Estimated output file size in bytes (for file rolling)</td></tr> |
| <tr><td><code>close()</code></td><td><code>io::Result<()></code></td><td>Flush remaining data and write footer</td></tr> |
| <tr><td><code>num_row_groups()</code></td><td><code>usize</code></td><td>Number of row groups written (available after close)</td></tr> |
| <tr><td><code>row_group_stats(rg_index)</code></td><td><code>&[ColumnStats]</code></td><td>Column statistics for a row group (available after close)</td></tr> |
| </tbody> |
| </table> |
| |
| <h2>Rust: Reading a File</h2> |
| <p> |
| <code>MosaicReader</code> is generic over any <code>InputFile</code> implementation. |
| Implement the <code>InputFile</code> trait for your data source to read from |
| local files, memory-mapped buffers, remote storage, etc.: |
| </p> |
| <pre><code><span class="kw">use</span> mosaic_core::reader::InputFile; |
| |
| <span class="kw">pub trait</span> <span class="ty">InputFile</span> { |
| <span class="kw">fn</span> <span class="fn">read_at</span>(&<span class="kw">self</span>, offset: <span class="ty">u64</span>, buf: &<span class="kw">mut</span> [<span class="ty">u8</span>]) -> io::Result<()>; |
| }</code></pre> |
| |
| <h3>1. Open and Inspect the Schema</h3> |
| <p> |
| Pass your <code>InputFile</code> implementation and the file length directly |
| to <code>MosaicReader::new</code>: |
| </p> |
| <pre><code><span class="kw">use</span> mosaic_core::reader::{MosaicReader, InputFile, ReaderAccess}; |
| |
| <span class="kw">let</span> reader = MosaicReader::new(my_input_file, file_len).unwrap(); |
| |
| <span class="cmt">// Iterate over all columns</span> |
| <span class="kw">for</span> col <span class="kw">in</span> &reader.schema().columns { |
| println!(<span class="str">"name={} type={:?} nullable={}"</span>, |
| col.name, col.data_type, col.nullable); |
| }</code></pre> |
| |
| <h3>ColumnMeta Fields</h3> |
| <table> |
| <thead> |
| <tr><th>Field</th><th>Type</th><th>Description</th></tr> |
| </thead> |
| <tbody> |
| <tr><td><code>name</code></td><td><code>String</code></td><td>Column name</td></tr> |
| <tr><td><code>data_type</code></td><td><code>arrow::datatypes::DataType</code></td><td>Arrow DataType (Int32, Utf8, Decimal128, Timestamp, etc.)</td></tr> |
| <tr><td><code>nullable</code></td><td><code>bool</code></td><td>Whether column allows nulls</td></tr> |
| </tbody> |
| </table> |
| |
| <h3>2. Read Row Groups as Arrow RecordBatch</h3> |
| <p> |
| Each row group is read as an Arrow <code>RecordBatch</code> via |
| <code>read_columns()</code>. This returns fully-constructed Arrow arrays with |
| proper null handling and type mapping. |
| </p> |
| <pre><code><span class="kw">use</span> arrow::array::*; |
| |
| <span class="kw">for</span> rg_idx <span class="kw">in</span> <span class="num">0</span>..reader.num_row_groups() { |
| <span class="kw">let</span> <span class="kw">mut</span> rg = reader.row_group_reader(rg_idx).unwrap(); |
| <span class="kw">let</span> batch = rg.read_columns().unwrap(); |
| |
| println!(<span class="str">"row group {} has {} rows, {} cols"</span>, |
| rg_idx, batch.num_rows(), batch.num_columns()); |
| |
| <span class="cmt">// Access columns by name</span> |
| <span class="kw">let</span> ages = batch.column_by_name(<span class="str">"age"</span>).unwrap() |
| .as_any().downcast_ref::<Int32Array>().unwrap(); |
| <span class="kw">let</span> names = batch.column_by_name(<span class="str">"name"</span>).unwrap() |
| .as_any().downcast_ref::<StringArray>().unwrap(); |
| |
| <span class="kw">for</span> i <span class="kw">in</span> <span class="num">0</span>..batch.num_rows() { |
| <span class="kw">if</span> !ages.is_null(i) { |
| println!(<span class="str">"age={} name={}"</span>, ages.value(i), names.value(i)); |
| } |
| } |
| }</code></pre> |
| |
| <h3>Projection Pushdown</h3> |
| <p> |
| Use <code>row_group_reader_projected</code> to read only specific columns. |
| Only the buckets containing the projected columns are decompressed — |
| reading 1 column out of 10,000 only touches 1 bucket instead of all 100. |
| The returned batch contains only the projected columns. |
| </p> |
| <pre><code><span class="kw">let</span> name_col = reader.schema().columns.iter() |
| .position(|c| c.name == <span class="str">"name"</span>).unwrap(); |
| <span class="kw">let</span> score_col = reader.schema().columns.iter() |
| .position(|c| c.name == <span class="str">"score"</span>).unwrap(); |
| |
| <span class="kw">let</span> <span class="kw">mut</span> rg = reader.row_group_reader_projected(rg_idx, &[name_col, score_col]).unwrap(); |
| <span class="kw">let</span> batch = rg.read_columns().unwrap(); |
| <span class="cmt">// batch contains only 2 columns: "name" and "score"</span> |
| <span class="cmt">// Use batch.schema().field(i).name() to identify columns</span></code></pre> |
| |
| <div class="note"> |
| <strong>Column ordering</strong> |
| The reader preserves the original schema column order. |
| Use <code>reader.schema().columns[i].name</code> or positional lookup to access columns. |
| </div> |
| |
| |
| <h2>Column Statistics (Filter Pushdown)</h2> |
| <p> |
| Enable per-column min/max statistics to allow query engines to skip entire row groups |
| whose value range does not match a filter predicate. |
| </p> |
| |
| <h3>Writing with Stats</h3> |
| <pre><code><span class="kw">let</span> <span class="kw">mut</span> writer = MosaicWriter::new(output, &arrow_schema, WriterOptions { |
| stats_columns: <span class="kw">vec!</span>[<span class="num">0</span>, <span class="num">2</span>], <span class="cmt">// build stats for columns 0 and 2</span> |
| ..Default::default() |
| })?;</code></pre> |
| |
| <h3>Getting Stats from Writer (after close)</h3> |
| <pre><code>writer.close()?; |
| |
| <span class="kw">for</span> rg_idx <span class="kw">in</span> <span class="num">0</span>..writer.num_row_groups() { |
| <span class="kw">let</span> stats = writer.row_group_stats(rg_idx); |
| <span class="kw">for</span> stat <span class="kw">in</span> stats { |
| println!(<span class="str">"col={} nulls={} min={:?} max={:?}"</span>, |
| stat.column_index, stat.null_count, stat.min, stat.max); |
| } |
| }</code></pre> |
| |
| <h3>Reading Stats from Reader</h3> |
| <pre><code><span class="kw">for</span> rg_idx <span class="kw">in</span> <span class="num">0</span>..reader.num_row_groups() { |
| <span class="kw">let</span> stats = reader.row_group_stats(rg_idx)?; |
| <span class="kw">for</span> stat <span class="kw">in</span> stats { |
| println!(<span class="str">"col={} nulls={} min={:?} max={:?}"</span>, |
| stat.column_index, stat.null_count, stat.min, stat.max); |
| } |
| }</code></pre> |
| <p> |
| Each <code>ColumnStats</code> contains: <code>column_index</code>, <code>null_count</code>, |
| <code>min: Option<Value></code>, <code>max: Option<Value></code>. |
| When all values in the row group are null, <code>min</code> and <code>max</code> are <code>None</code>. |
| </p> |
| |
| <h2>Running Tests</h2> |
| <pre><code><span class="cmt"># All Rust tests (core + ffi + jni)</span> |
| cargo test |
| |
| <span class="cmt"># Core only</span> |
| cargo test -p mosaic-core |
| |
| <span class="cmt"># Java compilation</span> |
| cd java && mvn compile</code></pre> |
| </div> |
| </main> |
| </body> |
| </html> |