blob: 5722e2a1e19014fb92685d0e379dee3826071220 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2022/07/25/apache-flink-kubernetes-operator-1.1.0-release-announcement/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="The community has continued to work hard on improving the Flink Kubernetes Operator capabilities since our first production ready release we launched about two months ago.
With the release of Flink Kubernetes Operator 1.1.0 we are proud to announce a number of exciting new features improving the overall experience of managing Flink resources and the operator itself in production environments.
Release Highlights # A non-exhaustive list of some of the more exciting features added in the release:">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Apache Flink Kubernetes Operator 1.1.0 Release Announcement" />
<meta property="og:description" content="The community has continued to work hard on improving the Flink Kubernetes Operator capabilities since our first production ready release we launched about two months ago.
With the release of Flink Kubernetes Operator 1.1.0 we are proud to announce a number of exciting new features improving the overall experience of managing Flink resources and the operator itself in production environments.
Release Highlights # A non-exhaustive list of some of the more exciting features added in the release:" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2022/07/25/apache-flink-kubernetes-operator-1.1.0-release-announcement/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2022-07-25T08:00:00+00:00" />
<meta property="article:modified_time" content="2022-07-25T08:00:00+00:00" />
<title>Apache Flink Kubernetes Operator 1.1.0 Release Announcement | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2022/07/25/apache-flink-kubernetes-operator-1.1.0-release-announcement/">Apache Flink Kubernetes Operator 1.1.0 Release Announcement</a>
</h1>
July 25, 2022 -
Gyula Fora
<a href="https://twitter.com/GyulaFora">(@GyulaFora)</a>
Matyas Orhidi
<p><p>The community has continued to work hard on improving the Flink Kubernetes Operator capabilities since our <a href="https://flink.apache.org/news/2022/06/05/release-kubernetes-operator-1.0.0.html">first production ready release</a> we launched about two months ago.</p>
<p>With the release of Flink Kubernetes Operator 1.1.0 we are proud to announce a number of exciting new features improving the overall experience of managing Flink resources and the operator itself in production environments.</p>
<h2 id="release-highlights">
Release Highlights
<a class="anchor" href="#release-highlights">#</a>
</h2>
<p>A non-exhaustive list of some of the more exciting features added in the release:</p>
<ul>
<li>Kubernetes Events on application and job state changes</li>
<li>New operator metrics</li>
<li>Unified and more robust reconciliation flow</li>
<li>Periodic savepoints</li>
<li>Custom Flink Resource Listeners</li>
<li>Dynamic watched namespaces</li>
<li>New built-in examples For Flink SQL and PyFlink</li>
<li>Experimental autoscaling support</li>
</ul>
<h3 id="kubernetes-events-for-application-and-job-state-changes">
Kubernetes Events for Application and Job State Changes
<a class="anchor" href="#kubernetes-events-for-application-and-job-state-changes">#</a>
</h3>
<p>The operator now emits native Kubernetes Events on relevant Flink Deployment and Job changes. This includes status changes, custom resource specification changes, deployment failures, etc.</p>
<pre tabindex="0"><code>Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Submit 53m JobManagerDeployment Starting deployment
Normal StatusChanged 52m Job Job status changed from RECONCILING to CREATED
Normal StatusChanged 52m Job Job status changed from CREATED to RUNNING
</code></pre><h3 id="new-operator-metrics">
New Operator Metrics
<a class="anchor" href="#new-operator-metrics">#</a>
</h3>
<p>The first version of the operator only came with basic system level metrics to monitor the JVM process.</p>
<p>In 1.1.0 we have introduced a wide range of additional metrics related to lifecycle-management, Kubernetes API server access and the Java Operator SDK framework the operator itself is built on. These metrics allow operator administrators to get a comprehensive view of what’s happening in the environment.</p>
<p>For details check the list of <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main/docs/operations/metrics-logging/#metrics">supported metrics</a>.</p>
<h3 id="unified-and-more-robust-reconciliation-flow">
Unified and more robust reconciliation flow
<a class="anchor" href="#unified-and-more-robust-reconciliation-flow">#</a>
</h3>
<p>We have refactored and streamlined the core reconciliation flow responsible for executing and tracking resource upgrades, savepoints, rollbacks and other operations.</p>
<p>In the process we made a number of important improvements to tolerate operator failures and temporary Kubernetes API outages more gracefully, which is critical in production environments.</p>
<h3 id="periodic-savepoints">
Periodic Savepoints
<a class="anchor" href="#periodic-savepoints">#</a>
</h3>
<p>By popular demand we have introduced periodic savepoints for applications and session jobs using a the following simple configuration option:</p>
<pre tabindex="0"><code>flinkConfiguration:
...
kubernetes.operator.periodic.savepoint.interval: 6h
</code></pre><p>Old savepoints are cleaned up automatically according to the user configured policy:</p>
<pre tabindex="0"><code>kubernetes.operator.savepoint.history.max.count: 5
kubernetes.operator.savepoint.history.max.age: 48h
</code></pre><h3 id="custom-flink-resource-listeners">
Custom Flink Resource Listeners
<a class="anchor" href="#custom-flink-resource-listeners">#</a>
</h3>
<p>The operator allows users to listen to events and status updates triggered for the Flink Resources managed by the operator.</p>
<p>This feature enables tighter integration with the user&rsquo;s own data platform. By implementing the <code>FlinkResourceListener</code> interface users can listen to both events and status updates per resource type (<code>FlinkDeployment</code> / <code>FlinkSessionJob</code>). The interface methods will be called after the respective events have been triggered by the system.</p>
<h3 id="new-sql-and-python-job-examples">
New SQL and Python Job Examples
<a class="anchor" href="#new-sql-and-python-job-examples">#</a>
</h3>
<p>To demonstrate the power of the operator for all Flink use-cases, we have added examples showcasing how to deploy Flink SQL and Python jobs.</p>
<p>We have also added a brief <a href="https://github.com/apache/flink-kubernetes-operator/tree/main/examples">README</a> for the examples to make it easier for you to find what you are looking for.</p>
<h3 id="dynamic-watched-namespaces">
Dynamic watched namespaces
<a class="anchor" href="#dynamic-watched-namespaces">#</a>
</h3>
<p>The operator can watch and manage custom resources in an arbitrary list of namespaces. The watched namespaces can be defined through the property <code>kubernetes.operator.watched.namespaces: ns1,ns2</code>. The list of watched namespaces can be changed anytime in the corresponding config map, however the operator ignores the changes unless dynamic watched namespaces is enabled.</p>
<p>This is controlled by the property <code>kubernetes.operator.dynamic.namespaces.enabled: true</code>.</p>
<h3 id="experimental-autoscaling-support">
Experimental autoscaling support
<a class="anchor" href="#experimental-autoscaling-support">#</a>
</h3>
<p>In this version we have taken the first steps toward enabling Kubernetes native autoscaling integration for the operator. The FlinkDeployment CRD now exposes the <code>scale</code> subresource which allows us to create HPA policies directly in Kubernetes that will monitor the task manager pods.</p>
<p>This integration is still very much experimental but we are planning to build on top of this in the upcoming releases to provide a reliable scaling mechanism.</p>
<p>You can find an example scaling policy <a href="https://github.com/apache/flink-kubernetes-operator/tree/main/examples#horizontal-pod-autoscaler">here</a>.</p>
<h2 id="whats-next">
What’s Next?
<a class="anchor" href="#whats-next">#</a>
</h2>
<p>In the coming months, our focus will be on the following key areas:</p>
<ul>
<li>Standalone deployment mode support</li>
<li>Hardening of rollback mechanism and stability conditions</li>
<li>Scaling improvements</li>
<li>Support for older Flink versions</li>
</ul>
<p>These features will allow the operator and users to benefit more from the recent advancements in Flink&rsquo;s scheduling capabilities.</p>
<h2 id="upgrading-to-110">
Upgrading to 1.1.0
<a class="anchor" href="#upgrading-to-110">#</a>
</h2>
<p>The new 1.1.0 release is backward compatible as long as you follow our <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-release-1.1/docs/operations/upgrade/#normal-upgrade-process">operator upgrade quide</a>.</p>
<p>Please ensure that <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-release-1.1/docs/operations/upgrade/#1-upgrading-the-crd">CRDs are updated</a> in order to enable some of the new features.</p>
<p>The upgrade should not impact any currently deployed Flink resources.</p>
<h2 id="release-resources">
Release Resources
<a class="anchor" href="#release-resources">#</a>
</h2>
<p>The source artifacts and helm chart are available on the Downloads page of the Flink website. You can easily try out the new features shipped in the official 1.1.0 release by following our <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-release-1.1/docs/try-flink-kubernetes-operator/quick-start/">quickstart guide</a>.</p>
<p>You can also find official Kubernetes Operator Docker images of the new version on <a href="https://hub.docker.com/r/apache/flink-kubernetes-operator">Dockerhub</a>.</p>
<p>For more details, check the <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-release-1.1/">updated documentation</a> and the <a href="https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12315522&amp;version=12351723">release notes</a>. We encourage you to download the release and share your feedback with the community through the Flink mailing lists or JIRA.</p>
<h2 id="list-of-contributors">
List of Contributors
<a class="anchor" href="#list-of-contributors">#</a>
</h2>
<p>Aitozi, Biao Geng, Chethan, ConradJam, Dora Marsal, Gyula Fora, Hao Xin, Hector Miuler Malpica Gallegos, Jaganathan Asokan, Jeesmon Jacob, Jim Busche, Maksim Aniskov, Marton Balassi, Matyas Orhidi, Nicholas Jiang, Peng Yuan, Peter Vary, Thomas Weise, Xin Hao, Yang Wang</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2022-07-25-release-kubernetes-operator-1.1.0.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li><a href="#release-highlights">Release Highlights</a>
<ul>
<li><a href="#kubernetes-events-for-application-and-job-state-changes">Kubernetes Events for Application and Job State Changes</a></li>
<li><a href="#new-operator-metrics">New Operator Metrics</a></li>
<li><a href="#unified-and-more-robust-reconciliation-flow">Unified and more robust reconciliation flow</a></li>
<li><a href="#periodic-savepoints">Periodic Savepoints</a></li>
<li><a href="#custom-flink-resource-listeners">Custom Flink Resource Listeners</a></li>
<li><a href="#new-sql-and-python-job-examples">New SQL and Python Job Examples</a></li>
<li><a href="#dynamic-watched-namespaces">Dynamic watched namespaces</a></li>
<li><a href="#experimental-autoscaling-support">Experimental autoscaling support</a></li>
</ul>
</li>
<li><a href="#whats-next">What’s Next?</a></li>
<li><a href="#upgrading-to-110">Upgrading to 1.1.0</a></li>
<li><a href="#release-resources">Release Resources</a></li>
<li><a href="#list-of-contributors">List of Contributors</a></li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>