blob: ccd60502de4e44d4a15f83185ae9ab9264e64947 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2022/08/29/apache-flink-table-store-0.2.0-release-announcement/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="The Apache Flink community is pleased to announce the release of the Apache Flink Table Store (0.2.0).
Please check out the full documentation for detailed information and user guides.
What is Flink Table Store # Flink Table Store is a data lake storage for streaming updates/deletes changelog ingestion and high-performance queries in real time.
As a new type of updatable data lake, Flink Table Store has the following features:">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Apache Flink Table Store 0.2.0 Release Announcement" />
<meta property="og:description" content="The Apache Flink community is pleased to announce the release of the Apache Flink Table Store (0.2.0).
Please check out the full documentation for detailed information and user guides.
What is Flink Table Store # Flink Table Store is a data lake storage for streaming updates/deletes changelog ingestion and high-performance queries in real time.
As a new type of updatable data lake, Flink Table Store has the following features:" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2022/08/29/apache-flink-table-store-0.2.0-release-announcement/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2022-08-29T08:00:00+00:00" />
<meta property="article:modified_time" content="2022-08-29T08:00:00+00:00" />
<title>Apache Flink Table Store 0.2.0 Release Announcement | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2022/08/29/apache-flink-table-store-0.2.0-release-announcement/">Apache Flink Table Store 0.2.0 Release Announcement</a>
</h1>
August 29, 2022 -
Jingsong Lee
<p><!-- improve style of tables -->
<style>
table { border: 0px solid black; table-layout: auto; width: 800px; }
th, td { border: 1px solid black; padding: 5px; padding-left: 10px; padding-right: 10px; }
th { text-align: center }
td { vertical-align: top }
</style>
<p>The Apache Flink community is pleased to announce the release of the
<a href="https://github.com/apache/flink-table-store">Apache Flink Table Store</a> (0.2.0).</p>
<p>Please check out the full <a href="//nightlies.apache.org/flink/flink-table-store-docs-release-0.2/">documentation</a> for detailed information and user guides.</p>
<br/>
<h2 id="what-is-flink-table-store">
What is Flink Table Store
<a class="anchor" href="#what-is-flink-table-store">#</a>
</h2>
<p>Flink Table Store is a data lake storage for streaming updates/deletes changelog ingestion and high-performance queries in real time.</p>
<center>
<img src="/img/blog/table-store/table-store-0.2.png" width="100%"/>
</center>
<p>As a new type of updatable data lake, Flink Table Store has the following features:</p>
<ul>
<li>Large throughput data ingestion while offering good query performance.</li>
<li>High performance query with primary key filters, as fast as 100ms.</li>
<li>Streaming reads are available on Lake Storage, lake storage can also be integrated
with Kafka to provide second-level streaming reads.</li>
</ul>
<br/>
<h2 id="notable-features">
Notable Features
<a class="anchor" href="#notable-features">#</a>
</h2>
<p>In this release, we have accomplished many exciting features.</p>
<h3 id="catalog">
Catalog
<a class="anchor" href="#catalog">#</a>
</h3>
<p>This release introduces Table Store&rsquo;s own catalog and supports automatic synchronization to the Hive Metastore.</p>
<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-sql" data-lang="sql"><span class="line"><span class="cl"><span class="k">CREATE</span><span class="w"> </span><span class="k">CATALOG</span><span class="w"> </span><span class="n">tablestore</span><span class="w"> </span><span class="k">WITH</span><span class="w"> </span><span class="p">(</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="s1">&#39;type&#39;</span><span class="o">=</span><span class="s1">&#39;table-store&#39;</span><span class="p">,</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="s1">&#39;warehouse&#39;</span><span class="o">=</span><span class="s1">&#39;hdfs://nn:8020/warehouse/path&#39;</span><span class="p">,</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="c1">-- optional hive metastore
</span></span></span><span class="line"><span class="cl"><span class="c1"></span><span class="w"> </span><span class="s1">&#39;metastore&#39;</span><span class="o">=</span><span class="s1">&#39;hive&#39;</span><span class="p">,</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="s1">&#39;uri&#39;</span><span class="o">=</span><span class="s1">&#39;thrift://&lt;hive-metastore-host-name&gt;:&lt;port&gt;&#39;</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"></span><span class="p">);</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"></span><span class="n">USE</span><span class="w"> </span><span class="k">CATALOG</span><span class="w"> </span><span class="n">tablestore</span><span class="p">;</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"></span><span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">my_table</span><span class="w"> </span><span class="p">...</span><span class="w">
</span></span></span></code></pre></div><h3 id="ecosystem">
Ecosystem
<a class="anchor" href="#ecosystem">#</a>
</h3>
<p>In this release, we provide support for Flink 1.14 and provide read support for multiple compute engines.</p>
<table>
<thead>
<tr>
<th>Engine</th>
<th>Version</th>
<th>Feature</th>
<th>Read Pushdown</th>
</tr>
</thead>
<tbody>
<tr>
<td>Flink</td>
<td>1.14</td>
<td>read, write</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Flink</td>
<td>1.15</td>
<td>read, write</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Hive</td>
<td>2.3</td>
<td>read</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Spark</td>
<td>2.4</td>
<td>read</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Spark</td>
<td>3.0</td>
<td>read</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Spark</td>
<td>3.1</td>
<td>read</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Spark</td>
<td>3.2</td>
<td>read</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Spark</td>
<td>3.3</td>
<td>read</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Trino</td>
<td>358</td>
<td>read</td>
<td>Projection, Filter</td>
</tr>
<tr>
<td>Trino</td>
<td>388</td>
<td>read</td>
<td>Projection, Filter</td>
</tr>
</tbody>
</table>
<br/>
<h3 id="append-only">
Append-only
<a class="anchor" href="#append-only">#</a>
</h3>
<p>The append-only table feature is a performance improvement and only accepts INSERT_ONLY data to append to the storage
instead of updating or de-duplicating the existing data, and hence suitable for use cases that do not
require updates (such as log data synchronization).</p>
<div class="highlight"><pre tabindex="0" class="chroma"><code class="language-sql" data-lang="sql"><span class="line"><span class="cl"><span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">my_table</span><span class="w"> </span><span class="p">(</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="p">...</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"></span><span class="p">)</span><span class="w"> </span><span class="k">WITH</span><span class="w"> </span><span class="p">(</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="s1">&#39;write-mode&#39;</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s1">&#39;append-only&#39;</span><span class="p">,</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"> </span><span class="p">...</span><span class="w">
</span></span></span><span class="line"><span class="cl"><span class="w"></span><span class="p">)</span><span class="w">
</span></span></span></code></pre></div><p>Streaming writing to an Append-only table also has asynchronous compaction, so you don&rsquo;t have to
worry about small files.</p>
<h3 id="rescale-bucket">
Rescale Bucket
<a class="anchor" href="#rescale-bucket">#</a>
</h3>
<p>Since the number of total buckets dramatically influences the performance, Table Store allows
users to tune bucket numbers by <code>ALTER TABLE</code> command and reorganize necessary partitions,
the old partitions remain unchanged.</p>
<br/>
<h2 id="getting-started">
Getting started
<a class="anchor" href="#getting-started">#</a>
</h2>
<p>Please refer to the <a href="//nightlies.apache.org/flink/flink-table-store-docs-release-0.2/docs/try-table-store/quick-start/">getting started guide</a> for more details.</p>
<br/>
<h2 id="whats-next">
What&rsquo;s Next?
<a class="anchor" href="#whats-next">#</a>
</h2>
<p>In the upcoming 0.3.0 release you can expect the following additional features:</p>
<ul>
<li>Streaming Changelog Concurrent Writes, the separation of Compaction.</li>
<li>Aggregation Table, to build your materialized view.</li>
<li>Changelog producing for partial-update/aggregation Tables.</li>
<li>Full Schema Evolution supports for drop column and rename column.</li>
<li>Lookup Supports for Flink Dim Join.</li>
</ul>
<p>Please give the release a try, share your feedback on the Flink mailing list and contribute to the project!</p>
<p>We encourage you to download the release and share your feedback with the community through the <a href="https://flink.apache.org/community.html#mailing-lists">Flink mailing lists</a>
or <a href="https://issues.apache.org/jira/issues/?jql=project%20%3D%20FLINK%20AND%20component%20%3D%20%22Table%20Store%22">JIRA</a>.</p>
<br/>
<h2 id="list-of-contributors">
List of Contributors
<a class="anchor" href="#list-of-contributors">#</a>
</h2>
<p>The Apache Flink community would like to thank every one of the contributors that have made this release possible:</p>
<p>Jane Chan, Jia Liu, Jingsong Lee, liliwei, Nicholas Jiang, openinx, tsreaper</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2022-08-29-release-table-store-0.2.0.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li><a href="#what-is-flink-table-store">What is Flink Table Store</a></li>
<li><a href="#notable-features">Notable Features</a>
<ul>
<li><a href="#catalog">Catalog</a></li>
<li><a href="#ecosystem">Ecosystem</a></li>
<li><a href="#append-only">Append-only</a></li>
<li><a href="#rescale-bucket">Rescale Bucket</a></li>
</ul>
</li>
<li><a href="#getting-started">Getting started</a></li>
<li><a href="#whats-next">What&rsquo;s Next?</a></li>
<li><a href="#list-of-contributors">List of Contributors</a></li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>