blob: ef714b32d7797503f338c06dfadc11c9d069564c [file] [log] [blame]
<!DOCTYPE html>
<html lang="en" dir=ZgotmplZ>
<head>
<link rel="stylesheet" href="/bootstrap/css/bootstrap.min.css">
<script src="/bootstrap/js/bootstrap.bundle.min.js"></script>
<link rel="stylesheet" type="text/css" href="/font-awesome/css/font-awesome.min.css">
<script src="/js/anchor.min.js"></script>
<script src="/js/flink.js"></script>
<link rel="canonical" href="https://flink.apache.org/how-to-contribute/code-style-and-quality-components/">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="Code Style and Quality Guide — Components Guide # Preamble # Pull Requests &amp; Changes # Common Coding Guide # Java Language Guide # Scala Language Guide # Components Guide # Formatting Guide # Component Specific Guidelines # Additional guidelines about changes in specific components.
Configuration Changes # Where should the config option go?
‘flink-conf.yaml’: All configuration that pertains to execution behavior that one may want to standardize across jobs.">
<meta name="theme-color" content="#FFFFFF"><meta property="og:title" content="Code Style and Quality Guide — Components Guide" />
<meta property="og:description" content="Code Style and Quality Guide — Components Guide # Preamble # Pull Requests &amp; Changes # Common Coding Guide # Java Language Guide # Scala Language Guide # Components Guide # Formatting Guide # Component Specific Guidelines # Additional guidelines about changes in specific components.
Configuration Changes # Where should the config option go?
‘flink-conf.yaml’: All configuration that pertains to execution behavior that one may want to standardize across jobs." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://flink.apache.org/how-to-contribute/code-style-and-quality-components/" /><meta property="article:section" content="how-to-contribute" />
<title>Code Style and Quality Guide — Components Guide | Apache Flink</title>
<link rel="manifest" href="/manifest.json">
<link rel="icon" href="/favicon.png" type="image/x-icon">
<link rel="alternate" hreflang="zh" href="https://flink.apache.org/zh/how-to-contribute/code-style-and-quality-components/" title="Apache Flink 代码样式和质量指南 — 组件">
<link rel="stylesheet" href="/book.min.22eceb4d17baa9cdc0f57345edd6f215a40474022dfee39b63befb5fb3c596b5.css" integrity="sha256-IuzrTRe6qc3A9XNF7dbyFaQEdAIt/uObY777X7PFlrU=">
<script defer src="/en.search.min.2698f0d1b683dae4d6cb071668b310a55ebcf1c48d11410a015a51d90105b53e.js" integrity="sha256-Jpjw0baD2uTWywcWaLMQpV688cSNEUEKAVpR2QEFtT4="></script>
<!--
Made with Book Theme
https://github.com/alex-shpak/hugo-book
-->
<meta name="generator" content="Hugo 0.124.1">
<script>
var _paq = window._paq = window._paq || [];
_paq.push(['disableCookies']);
_paq.push(["setDomains", ["*.flink.apache.org","*.nightlies.apache.org/flink"]]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="//analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '1']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</head>
<body dir=ZgotmplZ>
<header>
<nav class="navbar navbar-expand-xl">
<div class="container-fluid">
<a class="navbar-brand" href="/">
<img src="/img/logo/png/100/flink_squirrel_100_color.png" alt="Apache Flink" height="47" width="47" class="d-inline-block align-text-middle">
<span>Apache Flink</span>
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
<i class="fa fa-bars navbar-toggler-icon"></i>
</button>
<div class="collapse navbar-collapse" id="navbarSupportedContent">
<ul class="navbar-nav">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">About</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/what-is-flink/flink-architecture/">Architecture</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-applications/">Applications</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/flink-operations/">Operations</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/use-cases/">Use Cases</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/powered-by/">Powered By</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/roadmap/">Roadmap</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/community/">Community & Project Info</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/security/">Security</a>
</li>
<li>
<a class="dropdown-item" href="/what-is-flink/special-thanks/">Special Thanks</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Getting Started</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/try-flink/local_installation/">With Flink<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/docs/try-flink-kubernetes-operator/quick-start/">With Flink Kubernetes Operator<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable/docs/get-started/introduction/">With Flink CDC<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/docs/try-flink-ml/quick-start/">With Flink ML<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/getting-started/project-setup.html">With Flink Stateful Functions<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/docs/learn-flink/overview/">Training Course<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">Documentation</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-stable/">Flink 1.19 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-docs-master/">Flink Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-stable/">Kubernetes Operator 1.8 (latest)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-kubernetes-operator-docs-main">Kubernetes Operator Main (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-stable">CDC 3.0 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-cdc-docs-master">CDC Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-stable/">ML 2.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-ml-docs-master">ML Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-stable/">Stateful Functions 3.3 (stable)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
<li>
<a class="dropdown-item" href="https://nightlies.apache.org/flink/flink-statefun-docs-master">Stateful Functions Master (snapshot)<i class="link fa fa-external-link title" aria-hidden="true"></i>
</a>
</li>
</ul>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-bs-toggle="dropdown" aria-expanded="false">How to Contribute</a>
<ul class="dropdown-menu">
<li>
<a class="dropdown-item" href="/how-to-contribute/overview/">Overview</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-code/">Contribute Code</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/reviewing-prs/">Review Pull Requests</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/code-style-and-quality-preamble/">Code Style and Quality Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/contribute-documentation/">Contribute Documentation</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/documentation-style-guide/">Documentation Style Guide</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/improve-website/">Contribute to the Website</a>
</li>
<li>
<a class="dropdown-item" href="/how-to-contribute/getting-help/">Getting Help</a>
</li>
</ul>
</li>
<li class="nav-item">
<a class="nav-link" href="/posts/">Flink Blog</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/downloads/">Downloads</a>
</li>
</ul>
<div class="book-search">
<div class="book-search-spinner hidden">
<i class="fa fa-refresh fa-spin"></i>
</div>
<form class="search-bar d-flex" onsubmit="return false;"su>
<input type="text" id="book-search-input" placeholder="Search" aria-label="Search" maxlength="64" data-hotkeys="s/">
<i class="fa fa-search search"></i>
<i class="fa fa-circle-o-notch fa-spin spinner"></i>
</form>
<div class="book-search-spinner hidden"></div>
<ul id="book-search-results"></ul>
</div>
</div>
</div>
</nav>
<div class="navbar-clearfix"></div>
</header>
<main class="flex">
<section class="container book-page">
<article class="markdown"><h1 id="code-style-and-quality-guide--components-guide">
Code Style and Quality Guide — Components Guide
<a class="anchor" href="#code-style-and-quality-guide--components-guide">#</a>
</h1>
<h4 id="preamblehahahugoshortcode56s0hbhb">
<a href="/how-to-contribute/code-style-and-quality-preamble/">Preamble</a>
<a class="anchor" href="#preamblehahahugoshortcode56s0hbhb">#</a>
</h4>
<h4 id="pull-requests--changeshahahugoshortcode56s1hbhb">
<a href="/how-to-contribute/code-style-and-quality-pull-requests/">Pull Requests &amp; Changes</a>
<a class="anchor" href="#pull-requests--changeshahahugoshortcode56s1hbhb">#</a>
</h4>
<h4 id="common-coding-guidehahahugoshortcode56s2hbhb">
<a href="/how-to-contribute/code-style-and-quality-common/">Common Coding Guide</a>
<a class="anchor" href="#common-coding-guidehahahugoshortcode56s2hbhb">#</a>
</h4>
<h4 id="java-language-guidehahahugoshortcode56s3hbhb">
<a href="/how-to-contribute/code-style-and-quality-java/">Java Language Guide</a>
<a class="anchor" href="#java-language-guidehahahugoshortcode56s3hbhb">#</a>
</h4>
<h4 id="scala-language-guidehahahugoshortcode56s4hbhb">
<a href="/how-to-contribute/code-style-and-quality-scala/">Scala Language Guide</a>
<a class="anchor" href="#scala-language-guidehahahugoshortcode56s4hbhb">#</a>
</h4>
<h4 id="components-guidehahahugoshortcode56s5hbhb">
<a href="/how-to-contribute/code-style-and-quality-components/">Components Guide</a>
<a class="anchor" href="#components-guidehahahugoshortcode56s5hbhb">#</a>
</h4>
<h4 id="formatting-guidehahahugoshortcode56s6hbhb">
<a href="/how-to-contribute/code-style-and-quality-formatting/">Formatting Guide</a>
<a class="anchor" href="#formatting-guidehahahugoshortcode56s6hbhb">#</a>
</h4>
<h2 id="component-specific-guidelines">
Component Specific Guidelines
<a class="anchor" href="#component-specific-guidelines">#</a>
</h2>
<p><em>Additional guidelines about changes in specific components.</em></p>
<h3 id="configuration-changes">
Configuration Changes
<a class="anchor" href="#configuration-changes">#</a>
</h3>
<p>Where should the config option go?</p>
<ul>
<li>
<p><span style="text-decoration:underline;">‘flink-conf.yaml’:</span> All configuration that pertains to execution behavior that one may want to standardize across jobs. Think of it as parameters someone would set wearing an “ops” hat, or someone that provides a stream processing platform to other teams.</p>
</li>
<li>
<p><span style="text-decoration:underline;">‘ExecutionConfig’</span>: Parameters specific to an individual Flink application, needed by the operators during execution. Typical examples are watermark interval, serializer parameters, object reuse.</p>
</li>
<li>
<p><span style="text-decoration:underline;">ExecutionEnvironment (in code)</span>: Everything that is specific to an individual Flink application and is only needed to build program / dataflow, not needed inside the operators during execution.</p>
</li>
</ul>
<p>How to name config keys:</p>
<ul>
<li>
<p>Config key names should be hierarchical.
Think of the configuration as nested objects (JSON style)</p>
<pre tabindex="0"><code>taskmanager: {
jvm-exit-on-oom: true,
network: {
detailed-metrics: false,
request-backoff: {
initial: 100,
max: 10000
},
memory: {
fraction: 0.1,
min: 64MB,
max: 1GB,
buffers-per-channel: 2,
floating-buffers-per-gate: 16
}
}
}
</code></pre></li>
<li>
<p>The resulting config keys should hence be:</p>
<p><strong>NOT</strong> <code>&quot;taskmanager.detailed.network.metrics&quot;</code></p>
<p><strong>But rather</strong> <code>&quot;taskmanager.network.detailed-metrics&quot;</code></p>
</li>
</ul>
<h3 id="connectors">
Connectors
<a class="anchor" href="#connectors">#</a>
</h3>
<p>Connectors are historically hard to implement and need to deal with many aspects of threading, concurrency, and checkpointing.</p>
<p>As part of <a href="https://cwiki.apache.org/confluence/display/FLINK/FLIP-27%3A&#43;Refactor&#43;Source&#43;Interface">FLIP-27</a> we are working on making this much simpler for sources. New sources should not have to deal with any aspect of concurrency/threading and checkpointing any more.</p>
<p>A similar FLIP can be expected for sinks in the near future.</p>
<h3 id="examples">
Examples
<a class="anchor" href="#examples">#</a>
</h3>
<p>Examples should be self-contained and not require systems other than Flink to run. Except for examples that show how to use specific connectors, like the Kafka connector. Sources/sinks that are ok to use are <code>StreamExecutionEnvironment.socketTextStream</code>, which should not be used in production but is quite handy for exploring how things work, and file-based sources/sinks. (For streaming, there is the continuous file source)</p>
<p>Examples should also not be pure toy-examples but strike a balance between real-world code and purely abstract examples. The WordCount example is quite long in the tooth by now but it’s a good showcase of simple code that highlights functionality and can do useful things.</p>
<p>Examples should also be heavy in comments. They should describe the general idea of the example in the class-level Javadoc and describe what is happening and what functionality is used throughout the code. The expected input data and output data should also be described.</p>
<p>Examples should include parameter parsing, so that you can run an example (from the Jar that is created for each example using <code>bin/flink run path/to/myExample.jar --param1 … --param2</code>.</p>
<h3 id="table--sql-api">
Table &amp; SQL API
<a class="anchor" href="#table--sql-api">#</a>
</h3>
<h4 id="semantics">
Semantics
<a class="anchor" href="#semantics">#</a>
</h4>
<p><strong>The SQL standard should be the main source of truth.</strong></p>
<ul>
<li>Syntax, semantics, and features should be aligned with SQL!</li>
<li>We don’t need to reinvent the wheel. Most problems have already been discussed industry-wide and written down in the SQL standard.</li>
<li>We rely on the newest standard (SQL:2016 or ISO/IEC 9075:2016 when writing this document (<a href="https://standards.iso.org/ittf/PubliclyAvailableStandards/c065143_ISO_IEC_TR_19075-5_2016.zip">download</a>). Not every part is available online but a quick web search might help here.</li>
</ul>
<p>Discuss divergence from the standard or vendor-specific interpretations.</p>
<ul>
<li>Once a syntax or behavior is defined it cannot be undone easily.</li>
<li>Contributions that need to extent or interpret the standard need a thorough discussion with the community.</li>
<li>Please help committers by performing some initial research about how other vendors such as Postgres, Microsoft SQL Server, Oracle, Hive, Calcite, Beam are handling such cases.</li>
</ul>
<p>Consider the Table API as a bridge between the SQL and Java/Scala programming world.</p>
<ul>
<li>The Table API is an Embedded Domain Specific Language for analytical programs following the relational model.
It is not required to strictly follow the SQL standard in regards of syntax and names, but can be closer to the way a programming language would do/name functions and features, if that helps make it feel more intuitive.</li>
<li>The Table API might have some non-SQL features (e.g. map(), flatMap(), etc.) but should nevertheless “feel like SQL”. Functions and operations should have equal semantics and naming if possible.</li>
</ul>
<h4 id="common-mistakes">
Common mistakes
<a class="anchor" href="#common-mistakes">#</a>
</h4>
<ul>
<li>Support SQL’s type system when adding a feature.
<ul>
<li>A SQL function, connector, or format should natively support most SQL types from the very beginning.</li>
<li>Unsupported types lead to confusion, limit the usability, and create overhead by touching the same code paths multiple times.</li>
<li>For example, when adding a <code>SHIFT_LEFT</code> function, make sure that the contribution is general enough not only for <code>INT</code> but also <code>BIGINT</code> or <code>TINYINT</code>.</li>
</ul>
</li>
</ul>
<h4 id="testing">
Testing
<a class="anchor" href="#testing">#</a>
</h4>
<p>Test for nullability.</p>
<ul>
<li>SQL natively supports <code>NULL</code> for almost every operation and has a 3-valued boolean logic.</li>
<li>Make sure to test every feature for nullability as well.</li>
</ul>
<p>Avoid full integration tests</p>
<ul>
<li>Spawning a Flink mini-cluster and performing compilation of generated code for a SQL query is expensive.</li>
<li>Avoid integration tests for planner tests or variations of API calls.</li>
<li>Instead, use unit tests that validate the optimized plan which comes out of a planner. Or test the behavior of a runtime operator directly.</li>
</ul>
<h4 id="compatibility">
Compatibility
<a class="anchor" href="#compatibility">#</a>
</h4>
<p>Don’t introduce physical plan changes in patch releases!</p>
<ul>
<li>Backwards compatibility for state in streaming SQL relies on the fact that the physical execution plan remains stable. Otherwise the generated Operator Names/IDs change and state cannot be matched and restored.</li>
<li>Every bug fix that leads to changes in the optimized physical plan of a streaming pipeline hences breaks compatibility.</li>
<li>As a consequence, changes of the kind that lead to different optimizer plans can only be merged in major releases for now.</li>
</ul>
<h4 id="scala--java-interoperability-legacy-code-parts">
Scala / Java interoperability (legacy code parts)
<a class="anchor" href="#scala--java-interoperability-legacy-code-parts">#</a>
</h4>
<p>Keep Java in mind when designing interfaces.</p>
<ul>
<li>Consider whether a class will need to interact with a Java class in the future.</li>
<li>Use Java collections and Java Optional in interfaces for a smooth integration with Java code.</li>
<li>Don’t use features of case classes such as .copy() or apply() for construction if a class is subjected to be converted to Java.</li>
<li>Pure Scala user-facing APIs should use pure Scala collections/iterables/etc. for natural and idiomatic (“scalaesk”) integration with Scala.</li>
</ul>
</article>
<div class="edit-this-page">
<p>
<a href="https://cwiki.apache.org/confluence/display/FLINK/Flink+Translation+Specifications">Want to contribute translation?</a>
</p>
<p>
<a href="//github.com/apache/flink-web/edit/asf-site/docs/content/how-to-contribute/code-style-and-quality-components.md">
Edit This Page<i class="fa fa-edit fa-fw"></i>
</a>
</p>
</div>
</section>
<aside class="book-toc">
<nav id="TableOfContents"><h3>On This Page <a href="javascript:void(0)" class="toc" onclick="collapseToc()"><i class="fa fa-times" aria-hidden="true"></i></a></h3>
<ul>
<li><a href="#code-style-and-quality-guide--components-guide">Code Style and Quality Guide — Components Guide</a>
<ul>
<li>
<ul>
<li></li>
</ul>
</li>
<li><a href="#component-specific-guidelines">Component Specific Guidelines</a>
<ul>
<li><a href="#configuration-changes">Configuration Changes</a></li>
<li><a href="#connectors">Connectors</a></li>
<li><a href="#examples">Examples</a></li>
<li><a href="#table--sql-api">Table &amp; SQL API</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</nav>
</aside>
<aside class="expand-toc hidden">
<a class="toc" onclick="expandToc()" href="javascript:void(0)">
<i class="fa fa-bars" aria-hidden="true"></i>
</a>
</aside>
</main>
<footer>
<div class="separator"></div>
<div class="panels">
<div class="wrapper">
<div class="panel">
<ul>
<li>
<a href="https://flink-packages.org/">flink-packages.org</a>
</li>
<li>
<a href="https://www.apache.org/">Apache Software Foundation</a>
</li>
<li>
<a href="https://www.apache.org/licenses/">License</a>
</li>
<li>
<a href="/zh/how-to-contribute/code-style-and-quality-components/">
<i class="fa fa-globe" aria-hidden="true"></i>&nbsp;中文版
</a>
</li>
</ul>
</div>
<div class="panel">
<ul>
<li>
<a href="/what-is-flink/security">Security</a-->
</li>
<li>
<a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
</li>
<li>
<a href="https://www.apache.org/foundation/thanks.html">Thanks</a>
</li>
</ul>
</div>
<div class="panel icons">
<div>
<a href="/posts">
<div class="icon flink-blog-icon"></div>
<span>Flink blog</span>
</a>
</div>
<div>
<a href="https://github.com/apache/flink">
<div class="icon flink-github-icon"></div>
<span>Github</span>
</a>
</div>
<div>
<a href="https://twitter.com/apacheflink">
<div class="icon flink-twitter-icon"></div>
<span>Twitter</span>
</a>
</div>
</div>
</div>
</div>
<hr/>
<div class="container disclaimer">
<p>The contents of this website are © 2024 Apache Software Foundation under the terms of the Apache License v2. Apache Flink, Flink, and the Flink logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</p>
</div>
</footer>
</body>
</html>