blob: ec5a2b79851459755e0633116b106558f2768bb1 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2023/11/22/apache-flink-kubernetes-operator-1.7.0-release-announcement/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="The Apache Flink community is excited to announce the release of Flink Kubernetes Operator 1.7.0! The release introduces a large number of improvements to the autoscaler, including a complete decoupling from Kubernetes to support more Flink environments in the future. It&rsquo;s important to call out that the release explicitly drops support for Flink 1.13 and 1.14 as agreed by the community.
We encourage you to download the release and share your feedback with the community through the Flink mailing lists or JIRA!">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Apache Flink Kubernetes Operator 1.7.0 Release Announcement" />
<meta property="og:description" content="The Apache Flink community is excited to announce the release of Flink Kubernetes Operator 1.7.0! The release introduces a large number of improvements to the autoscaler, including a complete decoupling from Kubernetes to support more Flink environments in the future. It&rsquo;s important to call out that the release explicitly drops support for Flink 1.13 and 1.14 as agreed by the community.
We encourage you to download the release and share your feedback with the community through the Flink mailing lists or JIRA!" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2023/11/22/apache-flink-kubernetes-operator-1.7.0-release-announcement/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2023-11-22T08:00:00+00:00" />
<meta property="article:modified_time" content="2023-11-22T08:00:00+00:00" />
<title>Apache Flink Kubernetes Operator 1.7.0 Release Announcement | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2023/11/22/apache-flink-kubernetes-operator-1.7.0-release-announcement/">Apache Flink Kubernetes Operator 1.7.0 Release Announcement</a>
</h1>
November 22, 2023 -
Gyula Fora
<a href="https://twitter.com/GyulaFora">(@GyulaFora)</a>
Rui Fan
<a href="https://twitter.com/1996fanrui">(@1996fanrui)</a>
<p><p>The Apache Flink community is excited to announce the release of Flink Kubernetes Operator 1.7.0! The release introduces a large number of improvements to the autoscaler, including a complete decoupling from Kubernetes to support more Flink environments in the future. It&rsquo;s important to call out that the release explicitly drops support for Flink 1.13 and 1.14 as agreed by the community.</p>
<p>We encourage you to <a href="https://flink.apache.org/downloads.html">download the release</a> and share your feedback with the community through the Flink <a href="https://flink.apache.org/community.html#mailing-lists">mailing lists</a> or <a href="https://issues.apache.org/jira/browse/flink">JIRA</a>! We hope you like the new release and we’d be eager to learn about your experience with it.</p>
<h2 id="flink-version-support-policy-change">
Flink Version Support Policy Change
<a class="anchor" href="#flink-version-support-policy-change">#</a>
</h2>
<p>Previously the operator only added more and more supported Flink versions without a policy to remove support for these in the future. This resulted in a lot of legacy codepaths already in the core logic.</p>
<p>To keep technical debt at reasonable levels, the community decided to adopt a new Flink version support policy for the operator.</p>
<blockquote class="book-hint danger">
Starting from 1.7.0 the operator will only support the last 4 Flink minor versions corresponding to the date of the operator release.
For 1.7.0 this translates to: 1.18, 1.17, 1.16, 1.15
</blockquote>
<p>The operator will simply ignore changes made to resources with unsupported Flink versions. This also means that resources with unsupported versions are not possible to delete once the operator is upgraded. To temporarily work around this, users can upgrade the Flink version of the resource before deleting it.</p>
<h2 id="highlights">
Highlights
<a class="anchor" href="#highlights">#</a>
</h2>
<h3 id="decoupled-autoscaler-module">
Decoupled Autoscaler Module
<a class="anchor" href="#decoupled-autoscaler-module">#</a>
</h3>
<p>Starting from 1.7.0, the autoscaler logic is decoupled from Kubernetes and the Flink Kubernetes Operator.
The <code>flink-autoscaler</code> module now does not contain any Kubernetes related dependencies but defines a set of generic interfaces that are implemented by the operator.</p>
<p>As part of the decoupling effort, we released the initial version of the <code>Standalone Autoscaler</code> which serves as a limited alternative for anyone not using the Flink Kubernetes Operator currently. It supports scaling a single Flink cluster that can be any type, including: <code>Flink Standalone Cluster</code>, <code>MiniCluster</code>, <code>Flink YARN session cluster</code>, <code>Flink YARN application cluster</code>.</p>
<p>The Standalone Autoscaler runs as a separate Java Process. Please read the <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-release-1.7/docs/custom-resource/autoscaler/#autoscaler-standalone">Autoscaler Standalone</a> section for setup instructions. The standalone autoscaler is limited to Flink version 1.18.</p>
<p>To benefit from the best possible integration we recommend using the autoscaler as part of the Flink Kubernetes Operator. The standalone autoscaler is not planned to replace this either now or in the future.</p>
<p>To align with the new structure the autoscaler related configs will lose the <code>kubernetes.operator.</code> prefix going forward:</p>
<pre tabindex="0"><code># Old / Deprecated keys
# kubernetes.operator.job.autoscaler.enabled
# kubernetes.operator.job.autoscaler.metrics.window
# New Keys
job.autoscaler.enabled
job.autoscaler.metrics.window
</code></pre><p>Visit the <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-release-1.7/docs/custom-resource/autoscaler/#extensibility-of-autoscaler">Extensibility of Autoscaler</a> doc page to get more information.</p>
<h3 id="improved-source-metric-tracking">
Improved source metric tracking
<a class="anchor" href="#improved-source-metric-tracking">#</a>
</h3>
<p>Flink currently reports incorrectly low business metrics for sources that spend too much time fetching / polling input (for example IO bound sources).
This lead to the autoscaler not scaling sources that were actually running beyond their capacity.</p>
<p>To tackle this problem, we introduced a new mechanism in the autoscaler to automatically detect cases when the sources are running at full capacity (and backlog is building up). In these situations we switch to a new way to compute the maximum capacity (true processing rate) of the affected source vertices that is much more accurate in these cases. We refer to this mechanism currently as &ldquo;observed true processing rate&rdquo;, this feature is enabled by default and should not need any custom configuration.</p>
<h3 id="savepoint-triggering-improvements">
Savepoint triggering improvements
<a class="anchor" href="#savepoint-triggering-improvements">#</a>
</h3>
<p>To provide more flexibility to users, periodic savepoint triggering now supports configuring the trigger schedule using a Cron expression in Quartz format. You can find detailed info on the syntax <a href="http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html">here</a>.</p>
<h3 id="operator-rate-limiter">
Operator Rate Limiter
<a class="anchor" href="#operator-rate-limiter">#</a>
</h3>
<p>A small but operationally important change is that the operator now enables rate limiting for resource events by default. This helps work around some corner cases where the operator was previously overloading the API server on error loops.</p>
<p>The rate limiter is now enabled by default with the following config:</p>
<pre tabindex="0"><code>kubernetes.operator.rate-limiter.limit: 5
kubernetes.operator.rate-limiter.refresh-period: 15 s
</code></pre><h3 id="java-17-and-21-support">
Java 17 and 21 support
<a class="anchor" href="#java-17-and-21-support">#</a>
</h3>
<p>The operator can now be built and executed on Java 17 and 21 and we have enabled integration testing for these versions as well.</p>
<p>At the moment we are not releasing new operator docker images by the different Java versions, these need to be built and bundled by the users.
The official Kubernetes Operator image remains on Java 11.</p>
<h2 id="release-resources">
Release Resources
<a class="anchor" href="#release-resources">#</a>
</h2>
<p>The source artifacts and helm chart are available on the Downloads page of the Flink website. You can easily try out the new features shipped in the official 1.7.0 release by adding the Helm chart to your own local registry:</p>
<pre tabindex="0"><code>$ helm repo add flink-kubernetes-operator-1.7.0 https://archive.apache.org/dist/flink/flink-kubernetes-operator-1.7.0/
$ helm install flink-kubernetes-operator flink-kubernetes-operator-1.7.0/flink-kubernetes-operator --set webhook.create=false
</code></pre><p>You can also find official Kubernetes Operator Docker images of the new version on <a href="https://hub.docker.com/r/apache/flink-kubernetes-operator">Dockerhub</a>.</p>
<p>For more details, check the <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-release-1.7/">updated documentation</a> and the release notes. We encourage you to download the release and share your feedback with the community through the Flink mailing lists or JIRA.</p>
<h2 id="list-of-contributors">
List of Contributors
<a class="anchor" href="#list-of-contributors">#</a>
</h2>
<p>Alexander Fedulov, Clara Xiong, Daren Wong, Dongwoo Kim, Gabor Somogyi, Gyula Fora, Manan Mangal, Maximilian Michels, Nicolas Fraison, Peter Huang, Praneeth Ramesh, Rui Fan, Sergey Nuyanzin, SteNicholas, Zhanghao, Zhenqiu Huang, mehdid93</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2023-11-22-release-kubernetes-operator-1.7.0.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li><a href="#flink-version-support-policy-change">Flink Version Support Policy Change</a></li>
<li><a href="#highlights">Highlights</a>
<ul>
<li><a href="#decoupled-autoscaler-module">Decoupled Autoscaler Module</a></li>
<li><a href="#improved-source-metric-tracking">Improved source metric tracking</a></li>
<li><a href="#savepoint-triggering-improvements">Savepoint triggering improvements</a></li>
<li><a href="#operator-rate-limiter">Operator Rate Limiter</a></li>
<li><a href="#java-17-and-21-support">Java 17 and 21 support</a></li>
</ul>
</li>
<li><a href="#release-resources">Release Resources</a></li>
<li><a href="#list-of-contributors">List of Contributors</a></li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>