| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-0.93.0 plugin-docs plugin-id-default docs-doc-id-user-guide-tour"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.4.1"> |
| <title data-rh="true">Tour | Apache StreamPipes</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://streampipes.apache.org/img/favicon.png"><meta data-rh="true" name="twitter:image" content="https://streampipes.apache.org/img/favicon.png"><meta data-rh="true" property="og:url" content="https://streampipes.apache.org/docs/0.93.0/user-guide-tour/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="0.93.0"><meta data-rh="true" name="docusaurus_tag" content="docs-default-0.93.0"><meta data-rh="true" name="docsearch:version" content="0.93.0"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-0.93.0"><meta data-rh="true" property="og:title" content="Tour | Apache StreamPipes"><meta data-rh="true" name="description" content="StreamPipes is a framework that enables domain experts to model and execute stream processing pipelines in a big data infrastructure."><meta data-rh="true" property="og:description" content="StreamPipes is a framework that enables domain experts to model and execute stream processing pipelines in a big data infrastructure."><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://streampipes.apache.org/docs/0.93.0/user-guide-tour/"><link data-rh="true" rel="alternate" href="https://streampipes.apache.org/docs/0.93.0/user-guide-tour/" hreflang="en"><link data-rh="true" rel="alternate" href="https://streampipes.apache.org/docs/0.93.0/user-guide-tour/" hreflang="x-default"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache StreamPipes RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache StreamPipes Atom Feed"> |
| |
| |
| |
| <link rel="preconnect" href="//analytics.apache.org/"> |
| <script>var _paq=window._paq=window._paq||[];_paq.push(["disableCookies"]),_paq.push(["trackPageView"]),_paq.push(["enableLinkTracking"]),function(){var a="//analytics.apache.org/";_paq.push(["setTrackerUrl",a+"matomo.php"]),_paq.push(["setSiteId","35"]);var e=document,p=e.createElement("script"),t=e.getElementsByTagName("script")[0];p.type="text/javascript",p.async=!0,p.src=a+"matomo.js",t.parentNode.insertBefore(p,t)}()</script> |
| |
| |
| <link rel="stylesheet" href="/css/slick.min.css"> |
| <link rel="stylesheet" href="/css/slick-theme.min.css"> |
| <link rel="stylesheet" href="/css/fonts.css"> |
| <link rel="stylesheet" href="/css/admonition.css"> |
| <link rel="stylesheet" href="/css/custom.css"> |
| <link rel="stylesheet" href="/css/code-block-buttons.css"> |
| <link rel="stylesheet" href="/css/all.min.css"> |
| <link rel="stylesheet" href="/css/bootstrap.min.css"> |
| <link rel="stylesheet" href="/css/buttons.css"> |
| <link rel="stylesheet" href="/css/team.css"> |
| <link rel="stylesheet" href="/css/custom-website.css"> |
| <link rel="stylesheet" href="/css/page-banner.css"> |
| <link rel="stylesheet" href="/css/responsive.css"> |
| <script src="/js/jquery-3.3.1.min.js"></script> |
| <script src="/js/buttons.js"></script> |
| <script src="/js/slick.min.js"></script> |
| <script src="/js/custom.js"></script> |
| <script src="/js/clipboard.min.js"></script> |
| <script src="/js/code-block-buttons.js"></script><link rel="stylesheet" href="/assets/css/styles.a4107cef.css"> |
| <link rel="preload" href="/assets/js/runtime~main.7b6a994a.js" as="script"> |
| <link rel="preload" href="/assets/js/main.c3016da8.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" style="background-color:var(--color-accent);color:white" role="banner"><div class="content_knG7 announcementBarContent_xLdY">Apache StreamPipes 0.97.0 is available! ⭐️</div></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/sp-logo-color.png" alt="Apache StreamPipes" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/sp-logo-color.png" alt="Apache StreamPipes" class="themedImage_ToTc themedImage--dark_i4oU"></div></a></div><div class="navbar__items navbar__items--right"><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/">Home</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Docs</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/user-guide-introduction/">Apache StreamPipes - User Guide</a></li><li><a href="https://streampipes.apache.org/docs/docs/python/latest/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache StreamPipes - Python</a></li><li><a class="dropdown__link" href="/docs/faq-common-problems/">FAQ</a></li></ul></div><a class="navbar__item navbar__link" href="/download/">Download</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Resources</a><ul class="dropdown__menu"><li><a href="https://github.com/apache/streampipes/issues" target="_blank" rel="noopener noreferrer" class="dropdown__link">Issue Tracker<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://github.com/apache/streampipes/discussions" target="_blank" rel="noopener noreferrer" class="dropdown__link">Github Discussions Support<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="http://cwiki.apache.org/confluence/display/STREAMPIPES" target="_blank" rel="noopener noreferrer" class="dropdown__link">Developer wiki<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a class="dropdown__link" href="/resources/slides-videos/">Slides & Videos</a></li></ul></div><a class="navbar__item navbar__link" href="/blog/">Blog</a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Community</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/community/mailing-lists/">Mailing Lists</a></li><li><a class="dropdown__link" href="/community/team/">Team</a></li><li><a class="dropdown__link" href="/community/get-involved/">Get involved</a></li><li><a class="dropdown__link" href="/community/talks-events/">Talks & Events</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Apache</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Apache Software Foundation<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/how-it-works.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">How Apache Works<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/licenses" target="_blank" rel="noopener noreferrer" class="dropdown__link">License<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/security" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsoring Apache<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/apache/streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-github" aria-label="Apache StreamPipes Github"></a><a href="https://linkedin.com/company/apache-streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-linkedin" aria-label="Apache StreamPipes LinkedIn"></a><a href="https://twitter.com/streampipes" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-link h-twitter" aria-label="Apache StreamPipes Twitter"></a><div class="searchBox_ZlJk"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache StreamPipes<!-- --> <b>0.93.0</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/user-guide-tour/">latest version</a></b> (<!-- -->0.97.0<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.93.0</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Tour</h1></header><p>StreamPipes is a framework that enables domain experts to model and execute stream processing pipelines in a big data infrastructure. |
| The graphical user interface of StreamPipes is a web application that provides an easy to use solution for domain experts. |
| In this page, an overview of the many features of StreamPipes is given. We will tour through all features and explain what they do and how users can interact with them.</p><p>First of all, a user needs to log in to the system. |
| The credentials for the user are specified during the installation process.</p><p><img loading="lazy" alt="StreamPipes Login" src="/assets/images/login-ba98b3939647cb6ed7b6edb5f6581ea2.png" width="1200" height="720" class="img_ev3q"> </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="home">Home<a href="#home" class="hash-link" aria-label="Direct link to Home" title="Direct link to Home"></a></h2><p>After logging in, the user is redirected on to the home page. |
| The home page gives an overview over the different features available in the StreamPipes UI.</p><p>On the left, the navigation menu can be seen. |
| You can either use the icons on the left side or click on the menu icon on the |
| top left to open the details view of the navigation menu.</p><p>On the top right, a link refers to the documentation and the logout button is present.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/home/home.png" alt="Home" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/home/open_menu.png" alt="Open Menu" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pipeline-editor">Pipeline Editor<a href="#pipeline-editor" class="hash-link" aria-label="Direct link to Pipeline Editor" title="Direct link to Pipeline Editor"></a></h2><p>The first feature we are going to explain is the Pipeline Editor. |
| This is one of the central features of StreamPipes, since graphical modeling of pipelines takes place in this view.</p><p>On the top we can see four tabs: <strong>Data Sets</strong>, <strong>Data Streams</strong>, <strong>Processing Elements</strong>, and <strong>Data Sinks</strong>. |
| Each tab contains multiple pipeline elements, which can be installed at runtime. |
| The installation of pipeline elements is explained later in section <a href="/docs/0.93.0/user-guide-tour/#install-pipeline-elements">Install Pipeline Elements</a>.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/editor/2_PipelineEditor_DataStreams.png" alt="Data Streams" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/3_PipelineEditor_DataProcessors.png" alt="Processing Elements" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/4_PipelineEditor_DataSinks.png" alt="Data Sinks" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/1_PipelineEditor_DataSets.png" alt="Data Sets" class="img_ev3q"></div><p>There are multiple ways to search for a pipeline element. |
| The easiest way is to enter a search term in the search field on the top left corner. |
| The system filters the elements according to the search term and only presents the relevant ones. |
| Another way is to select one of the categories from the drop down menu. |
| The system then filters the elements according to the selected category. |
| The category of an element is defined by its functionality.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/editor/5_text_filter.png" alt="Text Filter" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/6_category_drop_down.png" alt="Category Drop Down" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/7_category_filter.png" alt="Category Filter" class="img_ev3q"></div><p>Modelling of a pipeline starts with choosing one ore more data sources. Therefore a data stream or data set must be selected |
| and moved into the editor via drag and drop. |
| After adding a data stream, we can select a processing element to transform the events of the data stream. |
| This is done again by dragging the processing element icon into our editor. |
| The mouse can be used to connect the two elements. |
| It is automatically checked in the background if the two elements are semantically compatible. |
| If this check is successful, a configuration menu is shown. It contains the parameters that can be modified by the |
| user. For all parameters, a description is provided and the system also prevents the user from entering parameters that |
| are not correct, according to the semantic description of the element. |
| It is also ensured that all required parameters are provided by the user, otherwise an error message is displayed.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/editor/8_connect_elements.png" alt="Connect Elements" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/9_configure_element.png" alt="Configure Elements" class="img_ev3q"></div><p>When the user tries to connect two elements that are not compatible, the system shows a <strong>connection error</strong> message. |
| The example illustrated below shows such a case. The user tried to connect a text filter to a light sensor. |
| This is not possible since the text filter processing element requires at least one event property of type string, which is not provided by the light sensor.</p><p><img loading="lazy" alt="Connection Error" src="/assets/images/10_connection_error-93521f25a4c91c65e281ebe9ba404cbd.png" width="1920" height="1121" class="img_ev3q"> </p><p>To further improve the usability, multiple ways are available to connect new elements besides the drag and drop option. |
| Each processing element has multiple short-cut buttons to connect it with another element. |
| The first one (eye icon) can be used to get a suggestion of all elements that are compatible with the current element. |
| The second one (plus icon) gives a recommendation on the elements the user might want to connect, based on the usage of the component in |
| other pipelines. |
| There is also a button (question tag icon) to open the documentation of a selected element. |
| Elements can be deleted by clicking the 'delete' button. |
| Each element can also be re-configured at a later point in time using the configuration button. |
| There is one important aspect about re-configuration of pipeline elements you need to be aware of: Only elements that are not yet connected to another element can be modified. |
| The reason for this behaviour is that some of the following elements might rely on the configuration of previous elements. |
| This way it is ensured that the user can not change the behaviour of the pipeline by accident.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/editor/11_shortcut_buttons.png" alt="Shortcut Buttons" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/12_compatible_elements.png" alt="Compatible Elements" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/13_recommend_elements.png" alt="Recommended elements" class="img_ev3q"></div><p>After the pipeline is completely modelled, the editor looks similar to the first image below. |
| Especially for larger pipelines, the auto layout button in the editor menu might be helpful. |
| With this button, the pipeline will be beautifully aligned in the editor, helping users to get a better overview of the complete pipeline. |
| On the top left corner of the editor, the 'save' button can be found. |
| After the modelling of the pipeline is done, use this button to save and execute the pipeline. |
| A save dialogue pops up when clicking the save button. |
| The pipeline title must be entered and an additional description can be provided. |
| It is recommended to always provide a description, because it makes it easier for other users to understand the meaning of the pipeline. |
| In the save menu, the user can either just store the pipeline configuration or store it and immediately start the pipeline. |
| Once the pipeline is executed, the user is redirected to the <em>Manage Pipeline</em> view. |
| In this view, the user gets immediate feedback whether all components did start correctly and the pipeline is up and running. |
| This view also shows that individual elements might run in different environments on different servers. |
| If there is an error during the execution, a notification containing a (hopefully) detailed error description is provided in this view.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/editor/14_save_adjust.png" alt="Save adjust delete" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/15_save_dialogue.png" alt="Complete pipeline" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/editor/16_pipeline_sucessfully_started.png" alt="Pipeline sucessfully started" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="connect-new-data-sources">Connect new Data Sources<a href="#connect-new-data-sources" class="hash-link" aria-label="Direct link to Connect new Data Sources" title="Direct link to Connect new Data Sources"></a></h2><p>With StreamPipes Connect it is possible to connect new data sources in StreamPipes with just a few clicks. |
| Therefore, we provide a marketplace with a lot of different adapters that can be configured and executed to create new <strong>Data Streams</strong> in the <strong>Pipeline Editor</strong>. |
| With StreamPipes Connect it is possible to connect specific data sources - e.g. an adapter streaming the current location of the ISS (International Space Station). |
| It is also possible to connect generic sources like message brokers or databases. |
| If the event schema of the data source is unknown, the system tries to infer the schema by extracting some sample data and analysing it.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/connect/new/01_connect.png" alt="Market Place" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/new/02_mqtt_adapter.png" alt="MQTT adapter" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/new/03_configure_mqtt.png" alt="Configure MQTT adpater" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/new/04_configure_format.png" alt="Guess event schema" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/new/05_guess_schema.png" alt="Guess event schema" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/new/06_edit_adapter.png" alt="Edit adapter" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/new/07_start_adapter.png" alt="Start adapter" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/new/08_data_stream_in_editor.png" alt="New data stram in pipeline editor" class="img_ev3q"></div><p>Additionally to connecting new sources, data can be cleaned, transformed, and enriched with meta-information. |
| Therefore, the event schema can be changed or enriched in step 3 (Define Event Schema).</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/connect/edit/01_edit_timestamp.png" alt="Edit a property to timestamp" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/edit/02_change_unit.png" alt="Change Unit of property" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/connect/edit/03_change_unit.png" alt="Change Unit of property" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="manage-pipelines">Manage Pipelines<a href="#manage-pipelines" class="hash-link" aria-label="Direct link to Manage Pipelines" title="Direct link to Manage Pipelines"></a></h2><p>The <em>Manage Pipelines</em> view provides an overview of all existing pipelines. |
| Existing pipelines can be managed using this view. |
| For instance, users can start and stop pipelines or delete them when they are not longer needed. |
| Pipeline actions can be performed by clicking one of the buttons next to each pipeline. |
| For each pipeline, the title and description is displayed in the table. |
| By clicking the edit symbol an overview of the created pipline is shown. In this window you are able to analyze statistics, identify errors or edit the pipeline.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/manage_pipelines/1_pipeline_overview.png" alt="Pipeline Overview" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/manage_pipelines/2_pipeline_information.png" alt="Pipeline infomation" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/manage_pipelines/3_delete_pipeline.png" alt="Delete Pipelines" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/manage_pipelines/4_pipeline_deleted.png" alt="Delete Pipelines" class="img_ev3q"></div><p>In a setting with many defined pipelines, it can get really hard to keep track of all pipelines. |
| This is why we introduce categories. |
| A category is a set of pipelines that can be defined by users in order to better organize pipelines. |
| By clicking on the "Manage Categories" button on the top left, a new category can be added to StreamPipes. |
| In this example, we create a new category named "New Demo Category". |
| After saving a category, pipelines can be added to the newly created category. |
| The new category is then presented as a tab in the <em>Pipeline Management</em> view. |
| This tab contains all previously defined pipelines.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/manage_pipelines/5_pipeline_category.png" alt="Pipeline Category" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/manage_pipelines/6_add_category.png" alt="Add Pipeline Category" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/manage_pipelines/7_new_category.png" alt="New Category" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/manage_pipelines/8_show_new_category.png" alt="Show new Category" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="live-dashboard">Live Dashboard<a href="#live-dashboard" class="hash-link" aria-label="Direct link to Live Dashboard" title="Direct link to Live Dashboard"></a></h2><p>The live dashboard can be used for multiple use cases. |
| It is a good way to live monitor a running system in a dashboard, but it can also be used during the pipeline development to get a fast feedback from newly created pipelines. |
| Below is a screenshot of an example dashboard showing the current value of pressure, which further shows a line chart and a trafficlight for a water level pipeline. |
| All pipelines that contain the "Dashboard Sink" can be visualized in this view. |
| To add a new visualisation, click on the "Add visualisation" button on the top left corner. |
| Afterwards, a three-step configuration menu is shown. |
| The first step is to select the pipeline that should be visualized. |
| In the second step, the type of visualization can be defined. |
| Currently 10 different types are available, including line charts, various map visualizations and data tables. |
| After selecting the type (in our example "Gauge"), you can select the specific measurement values of the data stream that should be displayed. |
| In the example below, the water level value should be monitored and the gauge value should range from 0 to 100. |
| Once all steps are completed, the new visualization is placed on the dashboard and live data is presented as soon as it becomes available.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/dashboard/01_dashboard.png" alt="Screenshot of live dashboard" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/dashboard/02_new_visualisation.png" alt="Select Pipeline to visualize" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/dashboard/03_select_gauge.png" alt="Select Visualization Type" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/dashboard/04_configure_gauge.png" alt="Configure Visualzation" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/dashboard/05_new_gauge_done.png" alt="Show new Visualzation" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="file-download">File Download<a href="#file-download" class="hash-link" aria-label="Direct link to File Download" title="Direct link to File Download"></a></h2><p>With the file download, it is possible to download stored files directly from Elasticsearch. |
| This can be very useful for example when a data dumb is needed for a specific time range. |
| All data that is written into Elasticsearch using the <em>Elasticsearch Sink</em> can be accessed by the file downloader. |
| A common use case is to download data for offline analysis and to train a machine learning algorithm. |
| First, an index must be defined, afterwards, the time range must be set. |
| A date picker helps users to enter the time range. |
| When a user clicks the "Create File" button the file is created. |
| All files stored on the server can be downloaded via the download button. |
| If the files are not longer needed, they can be deleted by clicking the delete button. |
| This will remove the file from the server. |
| Since data is stored in Elasticsearch anyways. it is recommended not to store the files for a longer period of time on the server. |
| When a file is needed again at a later point in time it is easy to create it again. |
| This way a lot of disk space on the server can be saved, especially when the files are rather large.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/file_download/01_file_download.png" alt="File Download" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/file_download/02_file_download.png" alt="File Download with selected time range" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/file_download/03_file_download.png" alt="File Download with new File" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="notifications">Notifications<a href="#notifications" class="hash-link" aria-label="Direct link to Notifications" title="Direct link to Notifications"></a></h2><p>Notifications are a good solution to notify someone when a urgent situation occurs. |
| A notification can be created by using the notification sink in the pipeline editor. |
| When using such a sink a configuration dialogue is presented to the user. |
| In this dialogue the user must provide enough information to resolve the solution when it occurs. |
| The message can be nicely configured in the message editor. |
| It is also possible to embed values of the event that triggered the notification. |
| This can be done with the #property# notation. |
| All available properties are presented in the notification configurator. |
| When the notification is triggered the #property# template is replaced with the actual value of the property.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/notification/01_select_notification_sink.png" alt="Select Notification" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/notification/02_configure_notification.png" alt="Configure Notification" class="img_ev3q"></div><p>A pop up icon on the notification tab in the menu shows the user how many unread notifications currently are in the system. |
| This icon also alerts users when new notifications occur. |
| In the notification overview all notifications are listed. |
| On the top are the new notifications that are not read yet. |
| A user can mark them as read by clicking on the little envelope icon. |
| Those notifications are then no longer in the unread section, but they remain in the "All Messages" view. |
| This way it is possible to keep track of all notifications and have a look at them at a later point in time.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/notification/03_new_notification.png" alt="New Notification" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="install-pipeline-elements">Install Pipeline Elements<a href="#install-pipeline-elements" class="hash-link" aria-label="Direct link to Install Pipeline Elements" title="Direct link to Install Pipeline Elements"></a></h2><p>StreamPipes is highly configurable and extensible. |
| Pipeline elements can be added and removed during runtime. |
| This can be done in the "Install Pipeline Elements" view. |
| All available pipeline elements are shown here and can be selected to install or uninstall. |
| It is also possible to select multiple or all of them and then install them all together. |
| When a new element is installed by the user it is automatically available in the "Pipeline Editor" and can be used for pipelines. |
| Elements that are uninstalled are removed from the system. |
| They can not longer be used within pipelines.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/install_elements/01_install_elements.png" alt="Install Elements View" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/install_elements/02_install_countAggregation.png" alt="Insall new Element" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/install_elements/03_installed_countAggregation.png" alt="Installation Successful" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/install_elements/04_countAggregation_in_editor.png" alt="Show new installed element" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="my-elements">My Elements<a href="#my-elements" class="hash-link" aria-label="Direct link to My Elements" title="Direct link to My Elements"></a></h2><p>The "My Elements" view gives a quick overview over all installed pipeline elements. |
| Here they can be inspected and the description can be re-imported. |
| In this view it is also possible to have a look at the JSON-LD representation of each element. |
| This is not important for a user, but it is worth noting that the system uses this machine understandable format to support the user. |
| For example all the information of the sources, like data schema or unit information is in the JSON-LD meta data.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/my_elements/01_my_elements.png" alt="My Elements Overview" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/my_elements/02_my_elements_jsonid.png" alt="View JSON-LD" class="img_ev3q"></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration">Configuration<a href="#configuration" class="hash-link" aria-label="Direct link to Configuration" title="Direct link to Configuration"></a></h2><p>The last feature is the <em>configuration view</em>. |
| Here, it is possible to change the configuration parameters of installed components. |
| All components containing processing elements automatically register in StreamPipes when they are started in the same network as the backend component. |
| Once a container is registered it is represented in the configuration view. |
| The green dot on the top left indicates that tha container is running properly. |
| When there is a problem with a container the green dot changes to red and the user knows there is a problem. |
| To configure the parameters the top right arrow of a configuration box must be clicked. |
| Then the configuration menu pops up. |
| Within there it is possible to change the parameters of a service. |
| To make the changes persistent the "Update" button must be clicked. |
| A user should keep in mind that sometimes it is necessary to restart a container when the parameters are changed. |
| Sometimes it is also necessary to re-import the pipeline element description, either by uninstalling and re-installing them after the container restart or be reloading the description in the "My elements" view.</p><div class="my-carousel"><img loading="lazy" src="/img/features_0_62_0/configuration/01_configuration_overview.png" alt="Configuration Overview" class="img_ev3q"><img loading="lazy" src="/img/features_0_62_0/configuration/02_change_config.png" alt="Change Configuration" class="img_ev3q"></div><p>Congratulations! You've just finished your first tour of StreamPipes. |
| Although there's still more to learn, we introduced most of the currently available features. |
| On the next page, the different processing elements that come with the installation are explained.</p></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"></div><div class="col lastUpdated_vwxv"><span class="theme-last-updated">Last updated<!-- --> on <b><time datetime="2023-11-27T14:58:33.000Z">Nov 27, 2023</time></b> by <b>Tim</b></span></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#home" class="table-of-contents__link toc-highlight">Home</a></li><li><a href="#pipeline-editor" class="table-of-contents__link toc-highlight">Pipeline Editor</a></li><li><a href="#connect-new-data-sources" class="table-of-contents__link toc-highlight">Connect new Data Sources</a></li><li><a href="#manage-pipelines" class="table-of-contents__link toc-highlight">Manage Pipelines</a></li><li><a href="#live-dashboard" class="table-of-contents__link toc-highlight">Live Dashboard</a></li><li><a href="#file-download" class="table-of-contents__link toc-highlight">File Download</a></li><li><a href="#notifications" class="table-of-contents__link toc-highlight">Notifications</a></li><li><a href="#install-pipeline-elements" class="table-of-contents__link toc-highlight">Install Pipeline Elements</a></li><li><a href="#my-elements" class="table-of-contents__link toc-highlight">My Elements</a></li><li><a href="#configuration" class="table-of-contents__link toc-highlight">Configuration</a></li></ul></div></div></div></div></main></div></div><footer class="container_yWIM"><div class="linksRow_XFIr"><div class="linksCol_rFJL"><div>ASF</div><ul><li class="footer__item"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer"><span></span><span>Foundation</span></a></li><li class="footer__item"><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer"><span></span><span>License</span></a></li><li class="footer__item"><a href="https://www.apache.org/events/" target="_blank" rel="noopener noreferrer"><span></span><span>Events</span></a></li><li class="footer__item"><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer"><span></span><span>Security</span></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer"><span></span><span>Sponsorship</span></a></li><li class="footer__item"><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer"><span></span><span>Thanks</span></a></li></ul></div><div class="linksCol_rFJL"><div>Community</div><ul><li class="footer__item"><a href="https://github.com/apache/streampipes/issues" target="_blank" rel="noopener noreferrer"><span></span><span>GitHub Issues</span></a></li><li class="footer__item"><a href="https://github.com/apache/streampipes/discussions" target="_blank" rel="noopener noreferrer"><span></span><span>Github Discussions</span></a></li><li class="footer__item"><a href="https://twitter.com/StreamPipes" target="_blank" rel="noopener noreferrer"><span></span><span>Twitter</span></a></li><li class="footer__item"><a href="https://www.linkedin.com/company/apache-streampipes" target="_blank" rel="noopener noreferrer"><span></span><span>LinkedIn</span></a></li></ul></div><div class="linksCol_rFJL"><div>More</div><ul><li class="footer__item"><a target="_parent" href="/blog/"><span></span><span>Blog</span></a></li><li class="footer__item"><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_parent" rel="noopener noreferrer"><span></span><span>Privacy</span></a></li></ul></div></div><div class="copyright_AoBa"><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer"><span style="display:inline-block;width:231.25px;height:40px"></span></a><div>Copyright © 2019-2025 The Apache Software Foundation. Apache StreamPipes, Apache, the Apache feather logo, and the Apache StreamPipes project logo are either registered trademarks or trademarks of the Apache Software Foundation.</div></div></footer></div> |
| <script src="/assets/js/runtime~main.7b6a994a.js"></script> |
| <script src="/assets/js/main.c3016da8.js"></script> |
| </body> |
| </html> |
