| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-0.93.0 plugin-docs plugin-id-default docs-doc-id-use-connect"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">StreamPipes Connect | Apache StreamPipes</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://streampipes.apache.org/img/favicon.png"><meta data-rh="true" name="twitter:image" content="https://streampipes.apache.org/img/favicon.png"><meta data-rh="true" property="og:url" content="https://streampipes.apache.org/docs/0.93.0/use-connect/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="0.93.0"><meta data-rh="true" name="docusaurus_tag" content="docs-default-0.93.0"><meta data-rh="true" name="docsearch:version" content="0.93.0"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-0.93.0"><meta data-rh="true" property="og:title" content="StreamPipes Connect | Apache StreamPipes"><meta data-rh="true" name="description" content="StreamPipes Connect is the module to connect external data sources with Apache StreamPipes directly from the user interface."><meta data-rh="true" property="og:description" content="StreamPipes Connect is the module to connect external data sources with Apache StreamPipes directly from the user interface."><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://streampipes.apache.org/docs/0.93.0/use-connect/"><link data-rh="true" rel="alternate" href="https://streampipes.apache.org/docs/0.93.0/use-connect/" hreflang="en"><link data-rh="true" rel="alternate" href="https://streampipes.apache.org/docs/0.93.0/use-connect/" hreflang="x-default"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache StreamPipes RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache StreamPipes Atom Feed"> |
| |
| |
| |
| <link rel="preconnect" href="//analytics.apache.org/"> |
| <script>var _paq=window._paq=window._paq||[];_paq.push(["disableCookies"]),_paq.push(["trackPageView"]),_paq.push(["enableLinkTracking"]),function(){var a="//analytics.apache.org/";_paq.push(["setTrackerUrl",a+"matomo.php"]),_paq.push(["setSiteId","35"]);var e=document,p=e.createElement("script"),t=e.getElementsByTagName("script")[0];p.type="text/javascript",p.async=!0,p.src=a+"matomo.js",t.parentNode.insertBefore(p,t)}()</script> |
| |
| |
| <link rel="stylesheet" href="/css/slick.min.css"> |
| <link rel="stylesheet" href="/css/slick-theme.min.css"> |
| <link rel="stylesheet" href="/css/fonts.css"> |
| <link rel="stylesheet" href="/css/admonition.css"> |
| <link rel="stylesheet" href="/css/custom.css"> |
| <link rel="stylesheet" href="/css/code-block-buttons.css"> |
| <link rel="stylesheet" href="/css/all.min.css"> |
| <link rel="stylesheet" href="/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/buttons.css"> |
| <link rel="stylesheet" href="/css/team.css"> |
| <link rel="stylesheet" href="/css/custom-website.css"> |
| <link rel="stylesheet" href="/css/page-banner.css"> |
| <link rel="stylesheet" href="/css/responsive.css"> |
| <script src="/js/jquery-3.3.1.min.js"></script> |
| <script src="/js/buttons.js"></script> |
| <script src="/js/slick.min.js"></script> |
| <script src="/js/custom.js"></script> |
| <script src="/js/clipboard.min.js"></script> |
| <script src="/js/code-block-buttons.js"></script><link rel="stylesheet" href="/assets/css/styles.a4107cef.css"> |
| <link rel="preload" href="/assets/js/runtime~main.7b6a994a.js" as="script"> |
| <link rel="preload" href="/assets/js/main.c3016da8.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:var(--color-accent);color:white" role="banner"><div class="content_knG7 announcementBarContent_xLdY">Apache StreamPipes 0.97.0 is available! ⭐️</div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/sp-logo-color.png" alt="Apache StreamPipes" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/sp-logo-color.png" alt="Apache StreamPipes" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/">Home</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Docs</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/user-guide-introduction/">Apache StreamPipes - User Guide</a></li><li><a href="https://streampipes.apache.org/docs/docs/python/latest/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache StreamPipes - Python</a></li><li><a class="dropdown__link" href="/docs/faq-common-problems/">FAQ</a></li></ul></div><a class="navbar__item navbar__link" href="/download/">Download</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Resources</a><ul class="dropdown__menu"><li><a href="https://github.com/apache/streampipes/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Issue Tracker<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://github.com/apache/streampipes/discussions" target="_blank" rel="noopener noreferrer" class="dropdown__link">Github Discussions Support<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="http://cwiki.apache.org/confluence/display/STREAMPIPES" target="_blank" rel="noopener noreferrer" class="dropdown__link">Developer wiki<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/resources/slides-videos/">Slides & Videos</a></li></ul></div><a class="navbar__item navbar__link" href="/blog/">Blog</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/community/mailing-lists/">Mailing Lists</a></li><li><a class="dropdown__link" href="/community/team/">Team</a></li><li><a class="dropdown__link" href="/community/get-involved/">Get involved</a></li><li><a class="dropdown__link" href="/community/talks-events/">Talks & Events</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">How Apache Works<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsoring Apache<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/apache/streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-github" aria-label="Apache StreamPipes Github"></a><a href="https://linkedin.com/company/apache-streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-linkedin" aria-label="Apache StreamPipes LinkedIn"></a><a href="https://twitter.com/streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-twitter" aria-label="Apache StreamPipes Twitter"></a><div class="searchBox_ZlJk"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="scroll-custom thin-scrollbar"><div class="custom-sidebar-version"><div class="custom-sidebar-inner"><span style="display:inline-block">Version</span> <div class="navbar__item dropdown dropdown--hoverable"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/docs/0.93.0/user-guide-introduction/">0.93.0</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/next/use-connect/">Next</a></li><li><a class="dropdown__link" href="/docs/use-connect/">0.97.0</a></li><li><a class="dropdown__link" href="/docs/0.95.1/use-connect/">0.95.1</a></li><li><a class="dropdown__link" href="/docs/0.95.0/use-connect/">0.95.0</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/docs/0.93.0/use-connect/">0.93.0</a></li><li><a class="dropdown__link" href="/docs/0.92.0/use-connect/">0.92.0</a></li><li><a class="dropdown__link" href="/docs/0.91.0/use-connect/">0.91.0</a></li></ul></div></div></div><div id="sidebarCssSelector"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/0.93.0/user-guide-introduction/">🚀 Try StreamPipes</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/0.93.0/introduction/">💡 Concepts</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/0.93.0/use-connect/">🎓 Use StreamPipes</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/0.93.0/use-connect/">StreamPipes Connect</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/0.93.0/use-pipeline-editor/">Pipeline Editor</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/0.93.0/use-managing-pipelines/">Managing Pipelines</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/0.93.0/use-dashboard/">Live Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/0.93.0/use-data-explorer/">Data Explorer</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/0.93.0/use-notifications/">Notifications</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/0.93.0/use-install-pipeline-elements/">Install Pipeline Elements</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/0.93.0/use-configurations/">Configurations</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/0.93.0/pe/org.apache.streampipes.connect.iiot.protocol.stream.kafka/">📚 Pipeline Elements</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/0.93.0/choosing-the-right-flavor/">⚡ Deploy StreamPipes</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/0.93.0/extend-setup/">💻 Customize StreamPipes</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/0.93.0/technicals-architecture/">🔧 Technicals</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/0.93.0/community-get-help/">👪 Community</a></div></li></ul></nav></div></div></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache StreamPipes<!-- --> <b>0.93.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/use-connect/">latest version</a></b> (<!-- -->0.97.0<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">🎓 Use StreamPipes</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">StreamPipes Connect</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.93.0</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>StreamPipes Connect</h1></header><p>StreamPipes Connect is the module to connect external data sources with Apache StreamPipes directly from the user interface. |
| StreamPipes Connect offers various adapters for common communication protocols and some specific sensors. Besides connecting data, StreamPipes Connect offers ways to pre-process data without the need to build pipelines and integrates a schema guesser that listens for incoming data and recommends the recognized event schema.</p><p>The screenshot below illustrates the data marketplace, which shown after navigating to "StreamPipes Connect" and then clicking the "New adapter" button at the top.</p><img loading="lazy" class="docs-image img_ev3q" src="/img/03_use-connect/01_connect-overview.png" alt="StreamPipes Connect Overview"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="connecting-new-data-sources">Connecting new data sources<a href="#connecting-new-data-sources" class="hash-link" aria-label="Direct link to Connecting new data sources" title="Direct link to Connecting new data sources"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="data-marketplace">Data Marketplace<a href="#data-marketplace" class="hash-link" aria-label="Direct link to Data Marketplace" title="Direct link to Data Marketplace"></a></h3><p>The data marketplace shows a list of all adapters that are currently installed in Apache StreamPipes. Each adapter offers various configuration options which depend on the specifics of the adapter. |
| Adapters are distinguished a) by the data source concept they provide (data set or data stream) and b) the adapter type, where we distinguish between <em>generic adapters</em>, which usually implement a generic communication protocol such as MQTT or Apache Kafka or a specific sensor interface (e.g., for Netio power sockets). |
| Several filter options are available to find a suitable adapter. The configuration of a new adapter starts with selecting one of the available adapters, which starts an assistant that supports the adapter generation.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="protocolbasic-settings">Protocol/Basic Settings<a href="#protocolbasic-settings" class="hash-link" aria-label="Direct link to Protocol/Basic Settings" title="Direct link to Protocol/Basic Settings"></a></h3><p>In the first step, basic configurations need to be provided. For instance, for an Apache PLC4X adapter, the IP address of the PLC needs to be provided. In this example, we provide basic settings for connecting to an Apache Kafka broker. After all values are provided, the "Next" button opens the next step. </p><img loading="lazy" class="docs-image img_ev3q" src="/img/03_use-connect/06_connect-create.png" alt="StreamPipes Connect Basic Settings"><h3 class="anchor anchorWithStickyNavbar_LWe7" id="format-specification">Format Specification<a href="#format-specification" class="hash-link" aria-label="Direct link to Format Specification" title="Direct link to Format Specification"></a></h3><p>The next step, format generation, is only available for generic adapters which support different message formats to be sent over the corresponding protocol. Think of a message broker that is able to consume messages in both JSON format or binary format. |
| Currently supported formats include XML, various JSON representations, images and CSV. After a format has been selected, further format configurations can be provided (depending on the selected format) to further customize the incoming message format.</p><img loading="lazy" class="docs-image img_ev3q" src="/img/03_use-connect/02_customize-format.png" alt="StreamPipes Connect Format Selection"><h3 class="anchor anchorWithStickyNavbar_LWe7" id="schema-editor">Schema Editor<a href="#schema-editor" class="hash-link" aria-label="Direct link to Schema Editor" title="Direct link to Schema Editor"></a></h3><p>In the next step, based on the previously provided protocol and format settings, the system will either provide the fixed/pre-defined schema of the adapter or, in case of specific adapters, will connect to the underlying system and try to listen for incoming data. After a few seconds, the schema editor will appear that provides a list of detected fields from the incoming events (the schema).</p><img loading="lazy" class="docs-image img_ev3q" src="/img/03_use-connect/03_schema-editor.png" alt="StreamPipes Connect Schema Editor"><p>In the toolbar, several configuration options are available which transform the original schema:</p><ul><li><strong>Add Nested Property</strong>. This option allows to modify the structure of the event by creating a nested structure. The schema can be simply changed by dragging and dropping fields into the nested structure.</li><li><strong>Add Static Value</strong>. This option allows to add a field containing a static value (e.g., an identifier) to the event.</li><li><strong>Add Timestamp</strong>. This options appends the current timestamp to each incoming event, useful in case the timestamp is not provided by the origin.</li><li><strong>Refresh</strong>. Re-triggers the schema guessing.</li><li><strong>Delete field</strong>. Select one or more fields by clicking the checkbox on the right and trigger the delete button.</li><li><strong>Property scope</strong>. For each field, a property scope can be defined which is either <em>Measurement</em>, <em>Dimension</em> or <em>Header</em>. These values are later be used in the pipeline editor to assist in configuring pipeline elements and do not have any functional consequence. |
| Use <em>Measurement</em> to indicate the field measures a value (e.g., a temperature value from a sensor), use <em>Dimension</em> for any identifier (e.g., the sensor ID) and use <em>Header</em> for any other metadata such as timestamps.</li></ul><p>For each field (also called event property) of the schema, additional configuration options are available by clicking the <em>Edit</em> button:</p><ul><li><strong>Label</strong>. Used to provide a human-readable label for the field, which will ease the identification of fields when building pipelines.</li><li><strong>Runtime Name.</strong> This is the identifier of the field in the underlying message representation format (e.g., the JSON key). Renaming the runtime name will trigger a so-called <em>transformation rule</em> which renames the incoming field name to the new field name before forwarding it to StreamPipes.</li><li><strong>Domain Property/Semantic Type</strong>. To help StreamPipes better understand the value which is represented by the field, semantic type information can be given. Up from StreamPipes 0.68.0, the semantic type can be selected from a wide range of available options. Additionally, an URL can be manually provided that indicates the meaning of the value (e.g., <a href="http://schema.org/Temperature" target="_blank" rel="noopener noreferrer">http://schema.org/Temperature</a>).</li><li><strong>Mark as Timestamp</strong>. Indicates that the selected value represents a timestamp. When selected, a <em>timestamp converter</em> can be configured which will convert incoming timestamps to the UNIX timestamp.</li><li><strong>Runtime Type</strong>. Here, the data type can be changed </li><li><strong>Unit</strong>. Allows to specify the unit in which the value is measured. Once selected, you can also automatically convert the unit to a target unit, which will then be inserted into the data stream produced by the adapter (see screenshot below). </li></ul><img loading="lazy" class="docs-image img_ev3q" src="/img/03_use-connect/04_schema-editor-conversion.png" alt="StreamPipes Connect Unit Conversion"><p>Assigning a timestamp is mandatory and can be either done by adding a timestamp from the menu, or by choosing an existing field and marking it as timestamp.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="adapter-generation">Adapter Generation<a href="#adapter-generation" class="hash-link" aria-label="Direct link to Adapter Generation" title="Direct link to Adapter Generation"></a></h3><p>Finally, the adapter is ready to be started. In the <em>Adapter Generation</em> page, a name and description for the resulting data stream must be provided. |
| Once started, StreamPipes creates your new adapter and displays a preview of the connected data, which refreshes about once per second. |
| Afterwards, the newly created data stream is available in the pipeline editor for further usage.</p><img loading="lazy" class="docs-image img_ev3q" src="/img/03_use-connect/05_adapter-generation.png" alt="StreamPipes Connect Adapter Generation"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="managing-adapters">Managing adapters<a href="#managing-adapters" class="hash-link" aria-label="Direct link to Managing adapters" title="Direct link to Managing adapters"></a></h2><p>Currently running adapters are available in the "Running adapters" section of StreamPipes Connect. Existing adapters can be stopped and deleted. Currently, there is no mechanism to edit an existing adapter or to stop the adapter without deleting it.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="adapter-templates">Adapter Templates<a href="#adapter-templates" class="hash-link" aria-label="Direct link to Adapter Templates" title="Direct link to Adapter Templates"></a></h3><p>For frequently used configurations, adapter templates can be created. An adapter template is a pre-configured adapter which can be further customized by users. Created adapter templates are available in the marketplace similar to standard adapters.</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"></div><div class="col lastUpdated_vwxv"><span class="theme-last-updated">Last updated<!-- --> on <b><time datetime="2023-11-27T14:58:33.000Z">Nov 27, 2023</time></b> by <b>Tim</b></span></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/0.93.0/concepts-overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/0.93.0/use-pipeline-editor/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Pipeline Editor</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#connecting-new-data-sources" class="table-of-contents__link toc-highlight">Connecting new data sources</a><ul><li><a href="#data-marketplace" class="table-of-contents__link toc-highlight">Data Marketplace</a></li><li><a href="#protocolbasic-settings" class="table-of-contents__link toc-highlight">Protocol/Basic Settings</a></li><li><a href="#format-specification" class="table-of-contents__link toc-highlight">Format Specification</a></li><li><a href="#schema-editor" class="table-of-contents__link toc-highlight">Schema Editor</a></li><li><a href="#adapter-generation" class="table-of-contents__link toc-highlight">Adapter Generation</a></li></ul></li><li><a href="#managing-adapters" class="table-of-contents__link toc-highlight">Managing adapters</a><ul><li><a href="#adapter-templates" class="table-of-contents__link toc-highlight">Adapter Templates</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="container_yWIM"><div class="linksRow_XFIr"><div class="linksCol_rFJL"><div>ASF</div><ul><li class="footer__item"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer"><span></span><span>Foundation</span></a></li><li class="footer__item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer"><span></span><span>License</span></a></li><li class="footer__item"><a href="https://www.apache.org/events/" target="_blank" rel="noopener noreferrer"><span></span><span>Events</span></a></li><li class="footer__item"><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer"><span></span><span>Security</span></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer"><span></span><span>Sponsorship</span></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer"><span></span><span>Thanks</span></a></li></ul></div><div class="linksCol_rFJL"><div>Community</div><ul><li class="footer__item"><a href="https://github.com/apache/streampipes/issues" target="_blank" rel="noopener noreferrer"><span></span><span>GitHub Issues</span></a></li><li class="footer__item"><a href="https://github.com/apache/streampipes/discussions" target="_blank" rel="noopener noreferrer"><span></span><span>Github Discussions</span></a></li><li class="footer__item"><a href="https://twitter.com/StreamPipes" target="_blank" rel="noopener noreferrer"><span></span><span>Twitter</span></a></li><li class="footer__item"><a href="https://www.linkedin.com/company/apache-streampipes" target="_blank" rel="noopener noreferrer"><span></span><span>LinkedIn</span></a></li></ul></div><div class="linksCol_rFJL"><div>More</div><ul><li class="footer__item"><a target="_parent" href="/blog/"><span></span><span>Blog</span></a></li><li class="footer__item"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_parent" rel="noopener noreferrer"><span></span><span>Privacy</span></a></li></ul></div></div><div class="copyright_AoBa"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer"><span style="display:inline-block;width:231.25px;height:40px"></span></a><div>Copyright © 2019-2025 The Apache Software Foundation. Apache StreamPipes, Apache, the Apache feather logo, and the Apache StreamPipes project logo are either registered trademarks or trademarks of the Apache Software Foundation.</div></div></footer></div> |
| <script src="/assets/js/runtime~main.7b6a994a.js"></script> |
| <script src="/assets/js/main.c3016da8.js"></script> |
| </body> |
| </html> |
