blob: bd82f7f2d20233eabc9392001bdf047f4fcba640 [file]
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Rust API - Mosaic</title>
<link rel="stylesheet" href="css/style.css">
<script src="js/main.js"></script>
</head>
<body>
<button class="menu-toggle" aria-label="Menu">&#9776;</button>
<div class="overlay"></div>
<aside class="sidebar">
<div class="sidebar-header">
<h2>Mosaic</h2>
<p>Columnar-bucket hybrid format</p>
</div>
<nav>
<ul>
<li><a href="index.html">Home</a></li>
<li><a href="design.html">Design</a></li>
<li><a href="rust-api.html" class="active">Rust API</a></li>
<li><a href="java-api.html">Java API</a></li>
<li><a href="python-api.html">Python API</a></li>
<li><a href="cpp-api.html">C++ API</a></li>
</ul>
</nav>
<div class="sidebar-footer">
<button class="theme-toggle">Dark Mode</button>
</div>
</aside>
<main class="main">
<div class="content">
<h1>Rust API</h1>
<p class="subtitle">Build from source and write your first Mosaic file in Rust.</p>
<h2>Building from Source</h2>
<pre><code>git clone &lt;repo-url&gt;
cd mosaic
cargo build</code></pre>
<p>Run the test suite to verify everything works:</p>
<pre><code>cargo test</code></pre>
<h2>Rust: Writing a File</h2>
<p>
Add <code>mosaic-core</code> as a dependency (path dependency within the workspace),
then use <code>MosaicWriter</code> to create a file. Data is written as Arrow
<code>RecordBatch</code> objects:
</p>
<pre><code><span class="kw">use</span> std::sync::Arc;
<span class="kw">use</span> arrow::datatypes::{DataType, Field, Schema};
<span class="kw">use</span> arrow::array::*;
<span class="kw">use</span> arrow::record_batch::RecordBatch;
<span class="kw">use</span> mosaic_core::writer::{MosaicWriter, WriterOptions};
<span class="kw">use</span> mosaic_core::spec::*;
<span class="cmt">// 1. Define an Arrow Schema</span>
<span class="kw">let</span> arrow_schema = Schema::new(<span class="kw">vec!</span>[
Field::new(<span class="str">"age"</span>, DataType::Int32, <span class="kw">true</span>),
Field::new(<span class="str">"name"</span>, DataType::Utf8, <span class="kw">true</span>),
Field::new(<span class="str">"score"</span>, DataType::Float64, <span class="kw">true</span>),
]);
<span class="cmt">// 2. Create writer (writes to any OutputFile implementation)</span>
<span class="kw">let</span> <span class="kw">mut</span> writer = MosaicWriter::new(output, &amp;arrow_schema, WriterOptions {
num_buckets: <span class="num">2</span>,
compression: COMPRESSION_ZSTD,
..Default::default()
})?;
<span class="cmt">// 3. Build an Arrow RecordBatch and write it</span>
<span class="kw">let</span> ages = Int32Array::from((<span class="num">0</span>..<span class="num">1000</span>).map(|i| <span class="num">20</span> + (i % <span class="num">50</span>)).collect::&lt;Vec&lt;_&gt;&gt;());
<span class="kw">let</span> names = StringArray::from((<span class="num">0</span>..<span class="num">1000</span>).map(|i| <span class="kw">format!</span>(<span class="str">"user_{}"</span>, i)).collect::&lt;Vec&lt;_&gt;&gt;());
<span class="kw">let</span> scores = Float64Array::from((<span class="num">0</span>..<span class="num">1000</span>).map(|i| i <span class="kw">as</span> <span class="ty">f64</span> * <span class="num">1.5</span>).collect::&lt;Vec&lt;_&gt;&gt;());
<span class="kw">let</span> batch = RecordBatch::try_new(
Arc::new(Schema::new(<span class="kw">vec!</span>[
Field::new(<span class="str">"age"</span>, DataType::Int32, <span class="kw">true</span>),
Field::new(<span class="str">"name"</span>, DataType::Utf8, <span class="kw">true</span>),
Field::new(<span class="str">"score"</span>, DataType::Float64, <span class="kw">true</span>),
])),
<span class="kw">vec!</span>[Arc::new(ages), Arc::new(names), Arc::new(scores)],
).unwrap();
writer.write_batch(&amp;batch).unwrap();
<span class="cmt">// 4. Finalize</span>
writer.close().unwrap();
<span class="cmt">// Estimated file size (for file rolling decisions)</span>
<span class="kw">let</span> est = writer.estimated_file_size();</code></pre>
<h3>Writer Methods</h3>
<table>
<thead>
<tr><th>Method</th><th>Return</th><th>Description</th></tr>
</thead>
<tbody>
<tr><td><code>write_batch(&amp;RecordBatch)</code></td><td><code>io::Result&lt;()&gt;</code></td><td>Write an Arrow RecordBatch</td></tr>
<tr><td><code>estimated_file_size()</code></td><td><code>u64</code></td><td>Estimated output file size in bytes (for file rolling)</td></tr>
<tr><td><code>close()</code></td><td><code>io::Result&lt;()&gt;</code></td><td>Flush remaining data and write footer</td></tr>
<tr><td><code>num_row_groups()</code></td><td><code>usize</code></td><td>Number of row groups written (available after close)</td></tr>
<tr><td><code>row_group_stats(rg_index)</code></td><td><code>&amp;[ColumnStats]</code></td><td>Column statistics for a row group (available after close)</td></tr>
</tbody>
</table>
<h2>Rust: Reading a File</h2>
<p>
<code>MosaicReader</code> is generic over any <code>InputFile</code> implementation.
Implement the <code>InputFile</code> trait for your data source to read from
local files, memory-mapped buffers, remote storage, etc.:
</p>
<pre><code><span class="kw">use</span> mosaic_core::reader::InputFile;
<span class="kw">pub trait</span> <span class="ty">InputFile</span> {
<span class="kw">fn</span> <span class="fn">read_at</span>(&amp;<span class="kw">self</span>, offset: <span class="ty">u64</span>, buf: &amp;<span class="kw">mut</span> [<span class="ty">u8</span>]) -&gt; io::Result&lt;()&gt;;
}</code></pre>
<h3>1. Open and Inspect the Schema</h3>
<p>
Pass your <code>InputFile</code> implementation and the file length directly
to <code>MosaicReader::new</code>:
</p>
<pre><code><span class="kw">use</span> mosaic_core::reader::{MosaicReader, InputFile, ReaderAccess};
<span class="kw">let</span> reader = MosaicReader::new(my_input_file, file_len).unwrap();
<span class="cmt">// Iterate over all columns</span>
<span class="kw">for</span> col <span class="kw">in</span> &amp;reader.schema().columns {
println!(<span class="str">"name={} type={:?} nullable={}"</span>,
col.name, col.data_type, col.nullable);
}</code></pre>
<h3>ColumnMeta Fields</h3>
<table>
<thead>
<tr><th>Field</th><th>Type</th><th>Description</th></tr>
</thead>
<tbody>
<tr><td><code>name</code></td><td><code>String</code></td><td>Column name</td></tr>
<tr><td><code>data_type</code></td><td><code>arrow::datatypes::DataType</code></td><td>Arrow DataType (Int32, Utf8, Decimal128, Timestamp, etc.)</td></tr>
<tr><td><code>nullable</code></td><td><code>bool</code></td><td>Whether column allows nulls</td></tr>
</tbody>
</table>
<h3>2. Read Row Groups as Arrow RecordBatch</h3>
<p>
Each row group is read as an Arrow <code>RecordBatch</code> via
<code>read_columns()</code>. This returns fully-constructed Arrow arrays with
proper null handling and type mapping.
</p>
<pre><code><span class="kw">use</span> arrow::array::*;
<span class="kw">for</span> rg_idx <span class="kw">in</span> <span class="num">0</span>..reader.num_row_groups() {
<span class="kw">let</span> <span class="kw">mut</span> rg = reader.row_group_reader(rg_idx).unwrap();
<span class="kw">let</span> batch = rg.read_columns().unwrap();
println!(<span class="str">"row group {} has {} rows, {} cols"</span>,
rg_idx, batch.num_rows(), batch.num_columns());
<span class="cmt">// Access columns by name</span>
<span class="kw">let</span> ages = batch.column_by_name(<span class="str">"age"</span>).unwrap()
.as_any().downcast_ref::&lt;Int32Array&gt;().unwrap();
<span class="kw">let</span> names = batch.column_by_name(<span class="str">"name"</span>).unwrap()
.as_any().downcast_ref::&lt;StringArray&gt;().unwrap();
<span class="kw">for</span> i <span class="kw">in</span> <span class="num">0</span>..batch.num_rows() {
<span class="kw">if</span> !ages.is_null(i) {
println!(<span class="str">"age={} name={}"</span>, ages.value(i), names.value(i));
}
}
}</code></pre>
<h3>Projection Pushdown</h3>
<p>
Use <code>row_group_reader_projected</code> to read only specific columns.
Only the buckets containing the projected columns are decompressed &mdash;
reading 1 column out of 10,000 only touches 1 bucket instead of all 100.
The returned batch contains only the projected columns.
</p>
<pre><code><span class="kw">let</span> name_col = reader.schema().columns.iter()
.position(|c| c.name == <span class="str">"name"</span>).unwrap();
<span class="kw">let</span> score_col = reader.schema().columns.iter()
.position(|c| c.name == <span class="str">"score"</span>).unwrap();
<span class="kw">let</span> <span class="kw">mut</span> rg = reader.row_group_reader_projected(rg_idx, &amp;[name_col, score_col]).unwrap();
<span class="kw">let</span> batch = rg.read_columns().unwrap();
<span class="cmt">// batch contains only 2 columns: "name" and "score"</span>
<span class="cmt">// Use batch.schema().field(i).name() to identify columns</span></code></pre>
<div class="note">
<strong>Column ordering</strong>
The reader preserves the original schema column order.
Use <code>reader.schema().columns[i].name</code> or positional lookup to access columns.
</div>
<h2>Column Statistics (Filter Pushdown)</h2>
<p>
Enable per-column min/max statistics to allow query engines to skip entire row groups
whose value range does not match a filter predicate.
</p>
<h3>Writing with Stats</h3>
<pre><code><span class="kw">let</span> <span class="kw">mut</span> writer = MosaicWriter::new(output, &amp;arrow_schema, WriterOptions {
stats_columns: <span class="kw">vec!</span>[<span class="num">0</span>, <span class="num">2</span>], <span class="cmt">// build stats for columns 0 and 2</span>
..Default::default()
})?;</code></pre>
<h3>Getting Stats from Writer (after close)</h3>
<pre><code>writer.close()?;
<span class="kw">for</span> rg_idx <span class="kw">in</span> <span class="num">0</span>..writer.num_row_groups() {
<span class="kw">let</span> stats = writer.row_group_stats(rg_idx);
<span class="kw">for</span> stat <span class="kw">in</span> stats {
println!(<span class="str">"col={} nulls={} min={:?} max={:?}"</span>,
stat.column_index, stat.null_count, stat.min, stat.max);
}
}</code></pre>
<h3>Reading Stats from Reader</h3>
<pre><code><span class="kw">for</span> rg_idx <span class="kw">in</span> <span class="num">0</span>..reader.num_row_groups() {
<span class="kw">let</span> stats = reader.row_group_stats(rg_idx)?;
<span class="kw">for</span> stat <span class="kw">in</span> stats {
println!(<span class="str">"col={} nulls={} min={:?} max={:?}"</span>,
stat.column_index, stat.null_count, stat.min, stat.max);
}
}</code></pre>
<p>
Each <code>ColumnStats</code> contains: <code>column_index</code>, <code>null_count</code>,
<code>min: Option&lt;Value&gt;</code>, <code>max: Option&lt;Value&gt;</code>.
When all values in the row group are null, <code>min</code> and <code>max</code> are <code>None</code>.
</p>
<h2>Running Tests</h2>
<pre><code><span class="cmt"># All Rust tests (core + ffi + jni)</span>
cargo test
<span class="cmt"># Core only</span>
cargo test -p mosaic-core
<span class="cmt"># Java compilation</span>
cd java &amp;&amp; mvn compile</code></pre>
</div>
</main>
</body>
</html>