| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-0.92.0 plugin-docs plugin-id-default docs-doc-id-dev-guide-introduction"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">Introduction | Apache StreamPipes</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://streampipes.apache.org/img/favicon.png"><meta data-rh="true" name="twitter:image" content="https://streampipes.apache.org/img/favicon.png"><meta data-rh="true" property="og:url" content="https://streampipes.apache.org/docs/0.92.0/dev-guide-introduction/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="0.92.0"><meta data-rh="true" name="docusaurus_tag" content="docs-default-0.92.0"><meta data-rh="true" name="docsearch:version" content="0.92.0"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-0.92.0"><meta data-rh="true" property="og:title" content="Introduction | Apache StreamPipes"><meta data-rh="true" name="description" content="StreamPipes is an open platform which can be easily extended at runtime by adding new data streams, data processors and data sinks."><meta data-rh="true" property="og:description" content="StreamPipes is an open platform which can be easily extended at runtime by adding new data streams, data processors and data sinks."><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://streampipes.apache.org/docs/0.92.0/dev-guide-introduction/"><link data-rh="true" rel="alternate" href="https://streampipes.apache.org/docs/0.92.0/dev-guide-introduction/" hreflang="en"><link data-rh="true" rel="alternate" href="https://streampipes.apache.org/docs/0.92.0/dev-guide-introduction/" hreflang="x-default"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache StreamPipes RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache StreamPipes Atom Feed"> |
| |
| |
| |
| <link rel="preconnect" href="//analytics.apache.org/"> |
| <script>var _paq=window._paq=window._paq||[];_paq.push(["disableCookies"]),_paq.push(["trackPageView"]),_paq.push(["enableLinkTracking"]),function(){var a="//analytics.apache.org/";_paq.push(["setTrackerUrl",a+"matomo.php"]),_paq.push(["setSiteId","35"]);var e=document,p=e.createElement("script"),t=e.getElementsByTagName("script")[0];p.type="text/javascript",p.async=!0,p.src=a+"matomo.js",t.parentNode.insertBefore(p,t)}()</script> |
| |
| |
| <link rel="stylesheet" href="/css/slick.min.css"> |
| <link rel="stylesheet" href="/css/slick-theme.min.css"> |
| <link rel="stylesheet" href="/css/fonts.css"> |
| <link rel="stylesheet" href="/css/admonition.css"> |
| <link rel="stylesheet" href="/css/custom.css"> |
| <link rel="stylesheet" href="/css/code-block-buttons.css"> |
| <link rel="stylesheet" href="/css/all.min.css"> |
| <link rel="stylesheet" href="/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/buttons.css"> |
| <link rel="stylesheet" href="/css/team.css"> |
| <link rel="stylesheet" href="/css/custom-website.css"> |
| <link rel="stylesheet" href="/css/page-banner.css"> |
| <link rel="stylesheet" href="/css/responsive.css"> |
| <script src="/js/jquery-3.3.1.min.js"></script> |
| <script src="/js/buttons.js"></script> |
| <script src="/js/slick.min.js"></script> |
| <script src="/js/custom.js"></script> |
| <script src="/js/clipboard.min.js"></script> |
| <script src="/js/code-block-buttons.js"></script><link rel="stylesheet" href="/assets/css/styles.a4107cef.css"> |
| <link rel="preload" href="/assets/js/runtime~main.85639ffd.js" as="script"> |
| <link rel="preload" href="/assets/js/main.386f6445.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:var(--color-accent);color:white" role="banner"><div class="content_knG7 announcementBarContent_xLdY">Apache StreamPipes 0.98.0 is available! ⭐️</div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/sp-logo-color.png" alt="Apache StreamPipes" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/sp-logo-color.png" alt="Apache StreamPipes" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/">Home</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Docs</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/user-guide-introduction/">Apache StreamPipes - User Guide</a></li><li><a href="https://streampipes.apache.org/docs/docs/python/latest/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache StreamPipes - Python</a></li><li><a class="dropdown__link" href="/docs/faq-common-problems/">FAQ</a></li></ul></div><a class="navbar__item navbar__link" href="/download/">Download</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Resources</a><ul class="dropdown__menu"><li><a href="https://github.com/apache/streampipes/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Issue Tracker<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://github.com/apache/streampipes/discussions" target="_blank" rel="noopener noreferrer" class="dropdown__link">Github Discussions Support<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="http://cwiki.apache.org/confluence/display/STREAMPIPES" target="_blank" rel="noopener noreferrer" class="dropdown__link">Developer wiki<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/resources/slides-videos/">Slides & Videos</a></li></ul></div><a class="navbar__item navbar__link" href="/blog/">Blog</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/community/mailing-lists/">Mailing Lists</a></li><li><a class="dropdown__link" href="/community/team/">Team</a></li><li><a class="dropdown__link" href="/community/get-involved/">Get involved</a></li><li><a class="dropdown__link" href="/community/talks-events/">Talks & Events</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">How Apache Works<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsoring Apache<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/apache/streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-github" aria-label="Apache StreamPipes Github"></a><a href="https://linkedin.com/company/apache-streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-linkedin" aria-label="Apache StreamPipes LinkedIn"></a><a href="https://twitter.com/streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-twitter" aria-label="Apache StreamPipes Twitter"></a><div class="searchBox_ZlJk"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache StreamPipes<!-- --> <b>0.92.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/user-guide-introduction/">latest version</a></b> (<!-- -->0.98.0<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.92.0</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Introduction</h1></header><p>StreamPipes is an open platform which can be easily extended at runtime by adding new data streams, data processors and data sinks. |
| When developing new components, usually multiple elements, called <em>pipeline elements</em>, are bundled into a self-contained <em>pipeline element container</em>. This container is deployed as a standalone microservice. This service is self-descriptive and exposes its description (see below for a detailed explanation) to the StreamPipes management backend. Once the service is deployed, it can be installed using the StreamPipes UI and all elements provided by this service are ready to be used as part of new pipelines.</p><p>To ease the extension of StreamPipes, we provide a powerful Software Development Kit (SDK) that allows you to define new pipeline elements for your StreamPipes instance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="concepts">Concepts<a href="#concepts" class="hash-link" aria-label="Direct link to Concepts" title="Direct link to Concepts"></a></h2><p>In this section, we briefly describe the main concepts of StreamPipes. Although it might give an abstract overview, we are sure that many concepts will be more clear once you've created your very first pipeline element yourself. So check out our tutorials!</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="data-stream">Data Stream<a href="#data-stream" class="hash-link" aria-label="Direct link to Data Stream" title="Direct link to Data Stream"></a></h3><p>A <em>Data Stream</em> is the main concept to describe the source of a pipeline. <em>DataStreams</em> consist of |
| an RDF <em>description</em> (which will be generated automatically when using the SDK) and a runtime |
| <em>implementation</em>. The description includes information on the <em>schema</em> of a data stream, e.g., measurement properties the payload of a stream provides. |
| Furthermore, the description contains information on the <em>grounding</em>, such as the transport format (e.g., JSON) and transport protoocol (e.g., MQTT or Kafka). |
| One or more data streams are assigned to a <em>Data Source</em> to improve discovery of existing streams.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="data-processor">Data Processor<a href="#data-processor" class="hash-link" aria-label="Direct link to Data Processor" title="Direct link to Data Processor"></a></h3><p><em>Data Processors</em> transform on or more input event streams to an output event stream. Data processors can be stateless (e.g., filter operations on every event of an input stream) or stateful (e.g., time-based aggregations using sliding windows). |
| Similar to data streams, processors consist of an RDF <em>description</em> and a corresponding <em>implementation</em>. The description is being used by the StreamPipes backend in order to determine the compatibility of a data processor and an input event stream and includes information the required minimum event schema as well as required user input and the definition of the output event stream.</p><p>The implementation of a data processor can be defined using a set of provided <em>runtime wrappers</em>. These wrappers define where computation logic actually takes place once a pipeline was started. We currently provide runtime wrappers for various Big Data processing engines (e.g., Apache Flink) and lightweight standalone processors.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="data-sink">Data Sink<a href="#data-sink" class="hash-link" aria-label="Direct link to Data Sink" title="Direct link to Data Sink"></a></h3><p>The concept of <em>Data Sinks</em> is very similar to the concept of data processors with the exception that sinks do not produce any output streams. |
| Therefore, sinks are used in StreamPipes to mark the end of a pipeline and reflect 3rd party applications, notifications or dashboard components.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="static-property">Static Property<a href="#static-property" class="hash-link" aria-label="Direct link to Static Property" title="Direct link to Static Property"></a></h3><p>Some data processors or data sinks might require input from users when pipelines are created using these elements. |
| For instance, a generic filter component might require information on the filter operation and a threshold value. |
| Such required user input can be modeled by defining <em>static properties</em>. Static properties can be defined in many ways, e.g., plain text input, selections (e.g., radio buttons) or can be linked to separately stored domain knowledge. |
| The SDK contains many convenient functions that help you defining static properties.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="output-strategy">Output Strategy<a href="#output-strategy" class="hash-link" aria-label="Direct link to Output Strategy" title="Direct link to Output Strategy"></a></h3><p>As mentioned above, data processors also define the output event schema. However, as data processors in StreamPipes are often generic and can therefore be linked to any event stream that matches the input requirement of a data processor, the exact output schema is not known in the development phase when a data processor is defined. |
| Therefore, data processors define their output using <em>output strategies</em>. Such strategies describe the transformation process, i.e., how an input stream is transformed to an output stream. |
| Multiple pre-defined output strategies exist that you can choose depending on the behaviour of a data processor. |
| For instance, the output schema of a filter component is usually similar to the input schema, so you would use a <em>KeepOutputStrategy</em>. |
| On the other hand, an enrichment component usually adds additional properties to an input schema - this can be defined using a <em>AppendOutputStrategy</em>. |
| Sometimes you want to let the user define the output schema. In this case, a <em>CustomOutputStrategy</em> can be defined.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="clients">Clients<a href="#clients" class="hash-link" aria-label="Direct link to Clients" title="Direct link to Clients"></a></h2><p>As stated in the beginning, pipeline element containers are deployed as self-contained microservices. The client types describes the environment this service is running in. |
| Currently supported clients are <em>standalone</em>, which defines a standalone service that contains both the description and implementation part (which is often submitted to a computing cluster prior to pipeline execution) in addition to an embedded Jetty web server which creates a fat jar file, and <em>embedded</em>, which creates a war file that can be imported into an existing application server.</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"></div><div class="col lastUpdated_vwxv"><span class="theme-last-updated">Last updated<!-- --> on <b><time datetime="2023-07-02T16:58:39.000Z">Jul 2, 2023</time></b> by <b>Dominik Riemer</b></span></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#concepts" class="table-of-contents__link toc-highlight">Concepts</a><ul><li><a href="#data-stream" class="table-of-contents__link toc-highlight">Data Stream</a></li><li><a href="#data-processor" class="table-of-contents__link toc-highlight">Data Processor</a></li><li><a href="#data-sink" class="table-of-contents__link toc-highlight">Data Sink</a></li><li><a href="#static-property" class="table-of-contents__link toc-highlight">Static Property</a></li><li><a href="#output-strategy" class="table-of-contents__link toc-highlight">Output Strategy</a></li></ul></li><li><a href="#clients" class="table-of-contents__link toc-highlight">Clients</a></li></ul></div></div></div></div></main></div></div><footer class="container_yWIM"><div class="linksRow_XFIr"><div class="linksCol_rFJL"><div>ASF</div><ul><li class="footer__item"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer"><span></span><span>Foundation</span></a></li><li class="footer__item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer"><span></span><span>License</span></a></li><li class="footer__item"><a href="https://www.apache.org/events/" target="_blank" rel="noopener noreferrer"><span></span><span>Events</span></a></li><li class="footer__item"><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer"><span></span><span>Security</span></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer"><span></span><span>Sponsorship</span></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer"><span></span><span>Thanks</span></a></li></ul></div><div class="linksCol_rFJL"><div>Community</div><ul><li class="footer__item"><a href="https://github.com/apache/streampipes/issues" target="_blank" rel="noopener noreferrer"><span></span><span>GitHub Issues</span></a></li><li class="footer__item"><a href="https://github.com/apache/streampipes/discussions" target="_blank" rel="noopener noreferrer"><span></span><span>Github Discussions</span></a></li><li class="footer__item"><a href="https://twitter.com/StreamPipes" target="_blank" rel="noopener noreferrer"><span></span><span>Twitter</span></a></li><li class="footer__item"><a href="https://www.linkedin.com/company/apache-streampipes" target="_blank" rel="noopener noreferrer"><span></span><span>LinkedIn</span></a></li></ul></div><div class="linksCol_rFJL"><div>More</div><ul><li class="footer__item"><a target="_parent" href="/blog/"><span></span><span>Blog</span></a></li><li class="footer__item"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_parent" rel="noopener noreferrer"><span></span><span>Privacy</span></a></li></ul></div></div><div class="copyright_AoBa"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer"><span style="display:inline-block;width:231.25px;height:40px"></span></a><div>Copyright © 2019-2025 The Apache Software Foundation. Apache StreamPipes, Apache, the Apache feather logo, and the Apache StreamPipes project logo are either registered trademarks or trademarks of the Apache Software Foundation.</div></div></footer></div> |
| <script src="/assets/js/runtime~main.85639ffd.js"></script> |
| <script src="/assets/js/main.386f6445.js"></script> |
| </body> |
| </html> |
