blob: f02cc60a995d94c3f877c439439f52959dd1a7f6 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en"><head>
<meta charset="utf-8">
<title>Obtain, Distribute and Use Temporary Credentials Automatically in Apache Spark and Apache Flink | Community Over Code Europe</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="description"
content="Community Over Code Europe is the annual gathering in Europe of the Apache Software Foundation community.">
<meta name="generator" content="Hugo 0.119.0"><meta property="og:title" content="Obtain, Distribute and Use Temporary Credentials Automatically in Apache Spark and Apache Flink" />
<meta property="og:description" content="The importance of security is increasing sharply nowadays which must be reflected in open source projects. Apache Spark and Apache Flink are two of the most widely used Big Data frameworks which can be used for data processing. Both of them offer dozens of external service connectors where authentication plays an essential role. Each external system does its authentication in a different way but a common framework can be provided to ease the life of developers." />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://eu.communityovercode.org/sessions/2024/obtain-distribute-and-use-temporary-credentials-automatically-in-apache-spark-and-apache-flink/" /><meta property="og:image" content="https://eu.communityovercode.org/images/card.jpg"/><meta property="article:section" content="sessions" />
<meta name="twitter:card" content="summary_large_image"/>
<meta name="twitter:image" content="https://eu.communityovercode.org/images/card.jpg"/>
<meta name="twitter:title" content="Obtain, Distribute and Use Temporary Credentials Automatically in Apache Spark and Apache Flink"/>
<meta name="twitter:description" content="The importance of security is increasing sharply nowadays which must be reflected in open source projects. Apache Spark and Apache Flink are two of the most widely used Big Data frameworks which can be used for data processing. Both of them offer dozens of external service connectors where authentication plays an essential role. Each external system does its authentication in a different way but a common framework can be provided to ease the life of developers."/>
<!-- plugins -->
<link rel="stylesheet" href="/plugins/bootstrap.min.css">
<link rel="stylesheet" href="/plugins/bootstrap-table.min.css">
<link rel="stylesheet" href="/plugins/fontawesome.css">
<!-- Main Stylesheet -->
<link rel="stylesheet" href='/scss/style.min.css?v=202406201555' media="screen">
<!--Favicon-->
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
</head><body class="interior">
<header class="header-bar">
<nav class="navbar navbar-expand-lg main-nav navbar-light fixed-top">
<a class="navbar-brand ml-4 pb-2" href='/'>
<img src="/images/coc-logo-color.svg" alt="Community Over Code Europe" class="img-fluid logo-b">
</a>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navigation"
aria-controls="navigation" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span>
</button>
<div class="collapse navbar-collapse text-center my-auto" id="navigation">
<ul class="navbar-nav me-auto align-items-center">
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
About
</a>
<div class="dropdown-menu">
<a class="dropdown-item" href="/about">Community Over Code</a>
<a class="dropdown-item" href="/about-the-asf">About the ASF</a>
<a class="dropdown-item" href="/diversity-and-inclusion">Diversity &amp; Inclusion</a>
</div>
</li>
<li class="nav-item">
<a class="nav-link" href="/program">Program</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/speakers">Speakers</a>
</li>
<li class="nav-item dropdown">
<a class="nav-link dropdown-toggle" href="#" role="button" data-toggle="dropdown" aria-haspopup="true"
aria-expanded="false">
Venue
</a>
<div class="dropdown-menu">
<a class="dropdown-item" href="/venue">About the venue</a>
<a class="dropdown-item" href="/how-to-get-there">How to get there</a>
</div>
</li>
<li class="nav-item">
<a class="nav-link" href="/#latest-news">News</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/faq">FAQ</a>
</li>
<li class="nav-item">
<a id="nav-button" href="/tickets" class="btn btn-orange text-white btn-rounded">Tickets</a>
</li>
</ul>
</div>
</nav>
</header>
<section class="page-header">
<div class="container">
<div class="row justify-content-center">
<div class="col-lg-8">
<div class="content text-center">
<h1 class="mb-3">Obtain, Distribute and Use Temporary Credentials Automatically in Apache Spark and Apache Flink</h1>
<div class="divider mx-auto mb-4 bg-secondary"></div>
</div>
</div>
</div>
</div>
</section>
<section class="speaker-detail">
<div class="container">
<div class="row mt-4">
<div class="image-column col-lg-3 d-none d-lg-block">
<div class="schedule-block col-lg-10 col-md-12 col-sm-12">
<div class="sec-title text-center">
<span class="title">Speaker(s):</span>
<div class="speaker-info" style="margin-bottom: 20px;">
<figure class="thumb my-3">
<a href="/speakers/gabor-somogyi/">
<div class="img-container">
<img src="/images/speakers/gabor-somogyi_hufc221596081cb83450c18e533df96a21_13822_400x0_resize_q75_h2_box.webp" alt="Photo of images/speakers/gabor-somogyi.jpg" class="img-fluid rounded-circle">
</div>
<h5 class="name">Gabor Somogyi</h5>
</a>
</figure>
</div>
</div>
</div>
</div>
<div class="info-column col-lg-9 col-md-12 col-sm-12">
<div class="inner-column">
<div class="text-box">
<div class="session-meta" id="date">
<div>
<em>Jun-05 14:40-15:10 in Symphony</em>
</div>
<div class="d-lg-none d-xl-none">
By
<a class="speaker-inline-item" href="https://eu.communityovercode.org/speakers/gabor-somogyi/">Gabor Somogyi</a>
</div>
<div class="content mt-4"><p>The importance of security is increasing sharply nowadays which must be reflected in open source projects. Apache Spark and Apache Flink are two of the most widely used Big Data frameworks which can be used for data processing. Both of them offer dozens of external service connectors where authentication plays an essential role. Each external system does its authentication in a different way but a common framework can be provided to ease the life of developers.</p>
<p>Introducing authentication creates many challenges for continuous data processing at high load. Credentials must be obtained in a timely and reliable manner which must be then propagated to several nodes. When too many nodes are authenticating at the same time then external services can be overloaded, which can cause temporary or permanent outages. A more reliable and less resource intensive solution was needed.</p>
<p>The mentioned problems have been addressed first in Apache Spark with a new delegation token framework which does all the heavy lifting. Building on the lessons learned in Spark a more generic implementation has been added to Apache Flink more recently. With these improvements workloads have been able to scale securely to thousands of workers. Attendees of this session will learn what are the common pitfalls of distributed system authentications and how to add such features easily to Apache Spark or Apache Flink.</p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
<footer>
<div class="container-fluid">
<div class="container py-5">
<div class="d-flex justify-content-between">
<div class="col-6 col-md-4 col-lg-3">
<div class="mb-3"> <img src="/images/logo-h.svg" class="img-fluid" alt="Community Over Code Europe"></div>
<ul class="list-inline mb-0">
<li class="list-inline-item mx-2 h3" data-toggle="tooltip" data-placement="top" title aria-label="Email us" data-original-title="Email us">
<a title="Email us" target="_blank" href="mailto:coceu@sg.com.mx?subject=[EU]">
<i class="fa fa-envelope" aria-hidden="true"></i>
</a>
</li>
<li class="list-inline-item mx-2 h3" data-toggle="tooltip" data-placement="top" title aria-label="Slack" data-original-title="Slack">
<a title="Slack" target="_blank" href="https://s.apache.org/apachecon-slack">
<i class="fab fa-slack" aria-hidden="true"></i>
</a>
</li>
<li class="list-inline-item mx-2 h3" data-toggle="tooltip" data-placement="top" title aria-label="Watch us on YouTube" data-original-title="Watch us on YouTube">
<a title="Watch us on YouTube" target="_blank" href="https://www.youtube.com/@communityovercode">
<i class="fab fa-youtube" aria-hidden="true"></i>
</a>
</li>
</ul>
</div>
<div class="col-md-6 text-right">
<div class="footer-links">
<ul>
<li><a href="/coc" >
Code of Conduct
</a></li>
<li><a href="/accessibility" >
Accessibility
</a></li>
<li><a href="/privacy" >
Privacy Policy
</a></li>
<li><a href="/team" >
Organizers
</a></li>
<li><a href="https://communityovercode.org/wp-content/uploads/2023/12/community-over-code-prospectus-2024.pdf" >
Prospectus
</a></li>
</ul>
</div>
</div>
</div>
</div>
</div>
<div class="footer-section footer-section__policies-section bg-dark">
<div class="container my-0 footer-section__policies-section--disclaimer">
Community Over Code operates under the terms of <a href="https://apache.org/foundation/policies/conduct">The ASF Code of Conduct</a>.
</div>
</div>
</footer>
<!-- JS Plugins -->
<script src="/plugins/jquery.min.js"></script>
<script src="/plugins/bootstrap.bundle.min.js"></script>
<script src="/plugins/bootstrap-table.min.js"></script>
<script src="/plugins/bootstrap-table.min.js"></script>
<script src="https://js.tito.io/v2"></script>
<script>
var _paq = window._paq = window._paq || [];
_paq.push(["disableCookies"]);
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://analytics.apache.org/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '39']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
</body>
</html>