blob: 1d3fcfdcfa8542206f43f6e084dd06d832f1cbf9 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/2023/05/17/apache-flink-kubernetes-operator-1.5.0-release-announcement/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="The Apache Flink community is excited to announce the release of Flink Kubernetes Operator 1.5.0! The release focuses on improvements to the job autoscaler that was introduced in the previous release and general operational hardening of the operator.
We encourage you to download the release and share your feedback with the community through the Flink mailing lists or JIRA! We hope you like the new release and we’d be eager to learn about your experience with it.">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Apache Flink Kubernetes Operator 1.5.0 Release Announcement" />
<meta property="og:description" content="The Apache Flink community is excited to announce the release of Flink Kubernetes Operator 1.5.0! The release focuses on improvements to the job autoscaler that was introduced in the previous release and general operational hardening of the operator.
We encourage you to download the release and share your feedback with the community through the Flink mailing lists or JIRA! We hope you like the new release and we’d be eager to learn about your experience with it." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/2023/05/17/apache-flink-kubernetes-operator-1.5.0-release-announcement/" /><meta property="article:section" content="posts" />
<meta property="article:published_time" content="2023-05-17T08:00:00+00:00" />
<meta property="article:modified_time" content="2023-05-17T08:00:00+00:00" />
<title>Apache Flink Kubernetes Operator 1.5.0 Release Announcement | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown">
<h1>
<a href="/2023/05/17/apache-flink-kubernetes-operator-1.5.0-release-announcement/">Apache Flink Kubernetes Operator 1.5.0 Release Announcement</a>
</h1>
May 17, 2023 -
Gyula Fora
<a href="https://twitter.com/GyulaFora">(@GyulaFora)</a>
<p><p>The Apache Flink community is excited to announce the release of Flink Kubernetes Operator 1.5.0! The release focuses on improvements to the job autoscaler that was introduced in the previous release and general operational hardening of the operator.</p>
<p>We encourage you to <a href="https://flink.apache.org/downloads.html">download the release</a> and share your feedback with the community through the Flink <a href="https://flink.apache.org/community.html#mailing-lists">mailing lists</a> or <a href="https://issues.apache.org/jira/browse/flink">JIRA</a>! We hope you like the new release and we’d be eager to learn about your experience with it.</p>
<h2 id="autoscaler-improvements">
Autoscaler improvements
<a class="anchor" href="#autoscaler-improvements">#</a>
</h2>
<h3 id="algorithm-improvements-and-better-scale-down-behaviour">
Algorithm improvements and better scale down behaviour
<a class="anchor" href="#algorithm-improvements-and-better-scale-down-behaviour">#</a>
</h3>
<p>The release contains important improvements to the core autoscaling logic. This includes improved stability of scaling decisions (leading to less parallelism oscillations) and better handling of slow or idle streams.</p>
<p>There are also some fixes related to output ratio computation and propagation that greatly improves the autoscaler on more complex streaming pipelines.</p>
<p>This version also introduces new metrics for tracking the number of scaling decisions and scaling errors together with some more Kubernetes events to improve the observability of the system.</p>
<h3 id="improved-default-configuration">
Improved default configuration
<a class="anchor" href="#improved-default-configuration">#</a>
</h3>
<p>We have simplified and improved some default autoscaler configs for a better out-of-the-box user experience.</p>
<p>Some notable changes:</p>
<pre tabindex="0"><code>kubernetes.operator.job.autoscaler.metrics.window: 5m -&gt; 10m
kubernetes.operator.job.autoscaler.target.utilization.boundary: 0.1 -&gt; 0.4
kubernetes.operator.job.autoscaler.scale-up.grace-period: 10m -&gt; 1h
kubernetes.operator.job.autoscaler.history.max.count: 1 -&gt; 3
kubernetes.operator.job.autoscaler.scaling.effectiveness.detection.enabled: true -&gt; false
kubernetes.operator.job.autoscaler.catch-up.duration: 10m -&gt; 5m
kubernetes.operator.job.autoscaler.restart.time: 5m -&gt; 3m
</code></pre><h2 id="crd-changes">
CRD Changes
<a class="anchor" href="#crd-changes">#</a>
</h2>
<h3 id="ephemeral-storage-support">
Ephemeral storage support
<a class="anchor" href="#ephemeral-storage-support">#</a>
</h3>
<p>Stateful streaming jobs often rely on ephemeral storage to store the working state of the pipeline. Previously it was only possible to change the ephemeral storage size through the pod template mechanism. The 1.5.0 release adds a new field to the task and jobmanager specification that allows configuring this similarly to other resources:</p>
<pre tabindex="0"><code>spec:
...
taskManager:
resource:
memory: &#34;2048m&#34;
cpu: 8
ephemeralStorage: &#34;10G&#34;
</code></pre><p>Make sure you upgrade the CRD together with the operator deployment to be able to access this feature. For more details check the <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main/docs/operations/upgrade/">docs</a></p>
<h2 id="general-operations">
General operations
<a class="anchor" href="#general-operations">#</a>
</h2>
<h3 id="fabric8-and-josdk-version-bump">
Fabric8 and JOSDK version bump
<a class="anchor" href="#fabric8-and-josdk-version-bump">#</a>
</h3>
<p>The operator have been updated to use the latest Java Operator SDK and Fabric8 versions that contain important fixes for production environments.</p>
<h3 id="health-probe-and-canary-resources">
Health probe and canary resources
<a class="anchor" href="#health-probe-and-canary-resources">#</a>
</h3>
<p>Previous operator versions already contained a rudimentary health probe to catch simple startup errors but did not have a good mechanism to catch errors that developed during the lifetime of the running operator.</p>
<p>The 1.5.0 version adds two significant improvements here:</p>
<ul>
<li>Improved health probe to detect informer errors after startup</li>
<li>Introduce canary resources for detecting general operator problems</li>
</ul>
<p>The new canary resource feature allows users to deploy special dummy resources (canaries) into selected namespaces. The operator health probe will then monitor that these resources are reconciled in a timely manner. This allows the operator health probe to catch any slowdowns, and other general reconciliation issues not covered otherwise.</p>
<p>Canary FlinkDeployment:</p>
<pre tabindex="0"><code>apiVersion: flink.apache.org/v1beta1
kind: FlinkDeployment
metadata:
name: canary
labels:
&#34;flink.apache.org/canary&#34;: &#34;true&#34;
</code></pre><h2 id="release-resources">
Release Resources
<a class="anchor" href="#release-resources">#</a>
</h2>
<p>The source artifacts and helm chart are available on the Downloads page of the Flink website. You can easily try out the new features shipped in the official 1.5.0 release by adding the Helm chart to your own local registry:</p>
<pre tabindex="0"><code>$ helm repo add flink-kubernetes-operator-1.5.0 https://archive.apache.org/dist/flink/flink-kubernetes-operator-1.5.0/
$ helm install flink-kubernetes-operator flink-kubernetes-operator-1.5.0/flink-kubernetes-operator --set webhook.create=false
</code></pre><p>You can also find official Kubernetes Operator Docker images of the new version on <a href="https://hub.docker.com/r/apache/flink-kubernetes-operator">Dockerhub</a>.</p>
<p>For more details, check the <a href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-release-1.5/">updated documentation</a> and the release notes. We encourage you to download the release and share your feedback with the community through the Flink mailing lists or JIRA.</p>
<h2 id="list-of-contributors">
List of Contributors
<a class="anchor" href="#list-of-contributors">#</a>
</h2>
<p>Gyula Fora, Marton Balassi, Mate Czagany, Maximilian Michels, Rafał Boniecki, Rodrigo Meneses, Tamir Sagi, Xin Hao, Xin Li, Zhanghao Chen, Zhenqiu Huang, Daren Wong, Gaurav Miglani, Peter Vary, Tan Kim, yangjf2019</p>
</p>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/posts/2023-05-17-release-kubernetes-operator-1.5.0.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li>
<ul>
<li><a href="#autoscaler-improvements">Autoscaler improvements</a>
<ul>
<li><a href="#algorithm-improvements-and-better-scale-down-behaviour">Algorithm improvements and better scale down behaviour</a></li>
<li><a href="#improved-default-configuration">Improved default configuration</a></li>
</ul>
</li>
<li><a href="#crd-changes">CRD Changes</a>
<ul>
<li><a href="#ephemeral-storage-support">Ephemeral storage support</a></li>
</ul>
</li>
<li><a href="#general-operations">General operations</a>
<ul>
<li><a href="#fabric8-and-josdk-version-bump">Fabric8 and JOSDK version bump</a></li>
<li><a href="#health-probe-and-canary-resources">Health probe and canary resources</a></li>
</ul>
</li>
<li><a href="#release-resources">Release Resources</a></li>
<li><a href="#list-of-contributors">List of Contributors</a></li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>