| <!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width,initial-scale=1"> <title>Getting Started :: Apache Hop (Incubating)</title> <link rel="canonical" href="https://hop.apache.org/manual/latest/getting-started.html"> <meta name="generator" content="Antora 2.3.4"> <link rel="stylesheet" href="../../_/css/site-8ea79c1e80.css"> </head> <body class="article"> <div class="cookie-banner" id="cookies-eu-banner" style="display: none;"> <div id="cookies-eu-content"> By continuing to visit this site, you accept the use of <a href="../../cookie-info/">cookies.</a> <button id="cookies-eu-accept">Accept</button> </div> <button id="cookies-eu-reject">Reject</button> </div> <header class="header"> <nav class="navbar"> <div class="navbar-brand"> <a class="nav-logo" href="../.."><span>Apache Hop (Incubating)</span></a> <div id="topbar-nav" class="navbar-menu"> <div class="navbar-end"> <a class="navbar-item" href="../..../../blog/">Blog</a> <div class="navbar-item has-dropdown is-hoverable"> <a class="navbar-link" href="#">Documentation</a> <div class="navbar-dropdown"> <a class="navbar-item" href="../../manual/latest/getting-started.html">Getting started</a> <a class="navbar-item" href="../../manual/latest/">User manual</a> <a class="navbar-item" href="../../tech-manual/latest/">Technical Documentation</a> <a class="navbar-item" href="../../dev-manual/latest/">Developer Documentation</a> <a class="navbar-item" href="../../docs/architecture/">Architecture</a> <a class="navbar-item" href="../../docs/roadmap/">Roadmap</a> <a class="navbar-item" href="../../docs/qa">Q&A</a> </div> </div> <div class="navbar-item has-dropdown is-hoverable"> <a class="navbar-link" href="#">Community</a> <div class="navbar-dropdown"> <a class="navbar-item" href="../../community/contributing/">Contributing</a> <a class="navbar-item" href="../../community/tools/">Tools</a> <a class="navbar-item" href="../../community/team/">Team</a> </div> </div> <a class="navbar-item" href="../..../../download/">Download</a> <div class="navbar-item has-dropdown is-hoverable"> <a class="navbar-link" href="#">About</a> <div class="navbar-dropdown"> <a class="navbar-item" href="https://www.apache.org/licenses/LICENSE-2.0">License</a> </div> </div> </div> </div> <div class="navbar-search"> <input id="search-input" type="text" placeholder="Search docs" class="ds-input" autocomplete="off" spellcheck="false" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-owns="algolia-autocomplete-listbox-0" dir="auto"> </div> <div class="navbar-tools"> <a href="https://chat.project-hop.org" title="Chat with us" target="_blank"><svg height="30" width="30" viewBox="0 0 30 30" class="brand-icon"><path d="m 23.399705,7.0517653 c -0.811583,-1.755522 -1.98643,-3.2324028 -3.542396,-4.4727237 0.04144,0.8420009 0.131904,2.6427217 0.131904,2.6427217 0,0 0.09696,0.1272 0.13632,0.182401 1.572478,2.199681 2.147037,4.6376017 1.623358,7.2857637 -1.050239,5.309843 -6.218105,8.549046 -11.455186,7.224805 C 5.6960779,18.752172 2.6254257,14.043529 3.4949926,9.3753663 4.1813918,5.6900843 6.3558852,3.2350424 9.8870488,1.9875216 l 0.1279682,-0.048 0.100416,-0.072 C 10.543401,1.2611208 10.961,0.6477608 11.406631,0 6.0547335,0.2656 0.67230798,4.5661633 0.05570071,11.107927 -0.54103457,17.438011 3.7200163,23.083134 9.8089048,24.321855 16.154114,25.612736 22.194522,21.863214 24.019864,15.86257 24.932631,12.862008 24.719032,9.9083263 23.399704,7.0542453 Z M 8.2979624,11.010647 c 0.267552,2.546882 2.4715176,4.282562 5.0209866,4.000322 2.504829,-0.2776 4.335931,-2.731601 3.861787,-5.2146427 -0.3696,-1.936001 -0.77808,-3.864402 -1.170046,-5.796084 -0.162144,-0.7991999 -0.326208,-1.5984007 -0.49824,-2.4414407 -0.06784,0.056 -0.09936,0.0848 -0.128064,0.1128 -0.858239,0.84272 -1.715997,1.6859209 -2.574236,2.5285617 -1.045056,1.02632 -2.08867,2.053921 -3.1362212,3.077522 -1.0506224,1.02664 -1.5288936,2.279601 -1.3760624,3.7327217 z"/></svg></a> <a href="https://twitter.com/projhop" title="Follow us on Twitter" target="_blank"><svg height="30" width="30" viewBox="0 0 30 30" class="brand-icon"><path d="M12 2c5.514 0 10 4.486 10 10s-4.486 10-10 10-10-4.486-10-10 4.486-10 10-10zm0-2c-6.627 0-12 5.373-12 12s5.373 12 12 12 12-5.373 12-12-5.373-12-12-12zm6.5 8.778c-.441.196-.916.328-1.414.388.509-.305.898-.787 1.083-1.362-.476.282-1.003.487-1.564.597-.448-.479-1.089-.778-1.796-.778-1.59 0-2.758 1.483-2.399 3.023-2.045-.103-3.86-1.083-5.074-2.572-.645 1.106-.334 2.554.762 3.287-.403-.013-.782-.124-1.114-.308-.027 1.14.791 2.207 1.975 2.445-.346.094-.726.116-1.112.042.313.978 1.224 1.689 2.3 1.709-1.037.812-2.34 1.175-3.647 1.021 1.09.699 2.383 1.106 3.773 1.106 4.572 0 7.154-3.861 6.998-7.324.482-.346.899-.78 1.229-1.274z"/></svg></a> <a href="https://www.linkedin.com/company/hop-project/" title="Follow us on LinkedIn" target="_blank"><svg height="30" width="30" viewBox="0 0 30 30" class="brand-icon"><path d="M19 0h-14c-2.761 0-5 2.239-5 5v14c0 2.761 2.239 5 5 5h14c2.762 0 5-2.239 5-5v-14c0-2.761-2.238-5-5-5zm-11 19h-3v-11h3v11zm-1.5-12.268c-.966 0-1.75-.79-1.75-1.764s.784-1.764 1.75-1.764 1.75.79 1.75 1.764-.783 1.764-1.75 1.764zm13.5 12.268h-3v-5.604c0-3.368-4-3.113-4 0v5.604h-3v-11h3v1.765c1.396-2.586 7-2.777 7 2.476v6.759z"/></svg></a> <a href="https://www.youtube.com/channel/UCGlcYslwe03Y2zbZ1W6DAGA" title="Subscribe to our YouTube channel" target="_blank"><svg height="30" width="30" viewBox="0 0 30 30" class="brand-icon"><path fill-rule="evenodd" d="M4.652 0h1.44l.988 3.702.916-3.702h1.454l-1.665 5.505v3.757h-1.431v-3.757l-1.702-5.505zm6.594 2.373c-1.119 0-1.861.74-1.861 1.835v3.349c0 1.204.629 1.831 1.861 1.831 1.022 0 1.826-.683 1.826-1.831v-3.349c0-1.069-.797-1.835-1.826-1.835zm.531 5.127c0 .372-.19.646-.532.646-.351 0-.554-.287-.554-.646v-3.179c0-.374.172-.651.529-.651.39 0 .557.269.557.651v3.179zm4.729-5.07v5.186c-.155.194-.5.512-.747.512-.271 0-.338-.186-.338-.46v-5.238h-1.27v5.71c0 .675.206 1.22.887 1.22.384 0 .918-.2 1.468-.853v.754h1.27v-6.831h-1.27zm2.203 13.858c-.448 0-.541.315-.541.763v.659h1.069v-.66c.001-.44-.092-.762-.528-.762zm-4.703.04c-.084.043-.167.109-.25.198v4.055c.099.106.194.182.287.229.197.1.485.107.619-.067.07-.092.105-.241.105-.449v-3.359c0-.22-.043-.386-.129-.5-.147-.193-.42-.214-.632-.107zm4.827-5.195c-2.604-.177-11.066-.177-13.666 0-2.814.192-3.146 1.892-3.167 6.367.021 4.467.35 6.175 3.167 6.367 2.6.177 11.062.177 13.666 0 2.814-.192 3.146-1.893 3.167-6.367-.021-4.467-.35-6.175-3.167-6.367zm-12.324 10.686h-1.363v-7.54h-1.41v-1.28h4.182v1.28h-1.41v7.54zm4.846 0h-1.21v-.718c-.223.265-.455.467-.696.605-.652.374-1.547.365-1.547-.955v-5.438h1.209v4.988c0 .262.063.438.322.438.236 0 .564-.303.711-.487v-4.939h1.21v6.506zm4.657-1.348c0 .805-.301 1.431-1.106 1.431-.443 0-.812-.162-1.149-.583v.5h-1.221v-8.82h1.221v2.84c.273-.333.644-.608 1.076-.608.886 0 1.18.749 1.18 1.631v3.609zm4.471-1.752h-2.314v1.228c0 .488.042.91.528.91.511 0 .541-.344.541-.91v-.452h1.245v.489c0 1.253-.538 2.013-1.813 2.013-1.155 0-1.746-.842-1.746-2.013v-2.921c0-1.129.746-1.914 1.837-1.914 1.161 0 1.721.738 1.721 1.914v1.656z"/></svg> <a href="https://github.com/project-hop/" title="Collaborate on GitHub"><svg class="brand-icon" viewBox="0 0 22 22"><path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path></svg></a> </a></div> <button class="navbar-burger" data-target="topbar-nav" type="button"> <span></span> <span></span> <span></span> </button> </div> </nav> </header> <div class="body"> <div class="nav-container" data-component="manual" data-version="latest"> <aside class="nav"> <div class="panels"> <div class="nav-panel-menu is-active" data-panel="menu"> <nav class="nav-menu"> <h3 class="title"><a href="index.html">User manual</a></h3> <ul class="nav-list"> <li class="nav-item" data-depth="0"> <ul class="nav-list"> <li class="nav-item is-current-page" data-depth="1"> <a class="nav-link" href="getting-started.html">Getting Started</a> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="concepts.html">Concepts</a> </li> <li class="nav-item" data-depth="1"> <button class="nav-item-toggle"></button> <a class="nav-link" href="hop-vs-kettle/index.html">Kettle → Hop</a> <ul class="nav-list"> <li class="nav-item" data-depth="2"> <a class="nav-link" href="hop-vs-kettle/hop-vs-kettle.html">Hop vs Kettle</a> </li> <li class="nav-item" data-depth="2"> <a class="nav-link" href="hop-vs-kettle/if-you-know-kettle.html">If You Know Kettle/PDI</a> </li> </ul> </li> <li class="nav-item" data-depth="1"> <button class="nav-item-toggle"></button> <a class="nav-link" href="hop-gui/index.html">Hop Gui</a> <ul class="nav-list"> <li class="nav-item" data-depth="2"> <a class="nav-link" href="hop-gui/environments/environments.html">Hop Environments</a> </li> <li class="nav-item" data-depth="2"> <button class="nav-item-toggle"></button> <a class="nav-link" href="hop-gui/pipelines/pipelines.html">Pipelines</a> <ul class="nav-list"> <li class="nav-item" data-depth="3"> <a class="nav-link" href="hop-gui/pipelines/hop-pipeline-editor.html">Pipeline Editor</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="hop-gui/pipelines/create-pipeline.html">Create a Pipeline</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="hop-gui/pipelines/run-preview-debug-pipeline.html">Run, Preview and Debug a Pipeline</a> </li> <li class="nav-item" data-depth="3"> <button class="nav-item-toggle"></button> <a class="nav-link" href="hop-gui/pipelines/pipeline-run-configurations/pipeline-run-configurations.html">Pipeline Run Configurations</a> <ul class="nav-list"> <li class="nav-item" data-depth="4"> <a class="nav-link" href="hop-gui/pipelines/pipeline-run-configurations/beam-dataflow-pipeline-engine.html">Beam Google DataFlow</a> </li> <li class="nav-item" data-depth="4"> <a class="nav-link" href="hop-gui/pipelines/pipeline-run-configurations/beam-direct-pipeline-engine.html">Beam Direct</a> </li> <li class="nav-item" data-depth="4"> <a class="nav-link" href="hop-gui/pipelines/pipeline-run-configurations/beam-flink-pipeline-engine.html">Beam Flink</a> </li> <li class="nav-item" data-depth="4"> <a class="nav-link" href="hop-gui/pipelines/pipeline-run-configurations/beam-spark-pipeline-engine.html">Beam Spark</a> </li> <li class="nav-item" data-depth="4"> <a class="nav-link" href="hop-gui/pipelines/pipeline-run-configurations/local-pipeline-engine.html">Local Native</a> </li> <li class="nav-item" data-depth="4"> <a class="nav-link" href="hop-gui/pipelines/pipeline-run-configurations/remote-pipeline-engine.html">Remote Native</a> </li> </ul> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="hop-gui/pipelines/metadata-injection.html">Metadata Injection</a> </li> </ul> </li> <li class="nav-item" data-depth="2"> <button class="nav-item-toggle"></button> <a class="nav-link" href="hop-gui/workflows/workflows.html">Workflows</a> <ul class="nav-list"> <li class="nav-item" data-depth="3"> <a class="nav-link" href="hop-gui/workflows/create-workflow.html">Create a Workflow</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="hop-gui/workflows/run-debug-workflow.html">Run and Debug a Workflow</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="hop-gui/workflows/workflow-run-configurations/workflow-run-configurations.html">Workflow Run Configurations</a> </li> </ul> </li> <li class="nav-item" data-depth="2"> <a class="nav-link" href="hop-gui/variables/variables.html">Variables</a> </li> </ul> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="hop-run/index.html">Hop Run</a> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="hop-server/index.html">Hop Server</a> </li> <li class="nav-item" data-depth="1"> <button class="nav-item-toggle"></button> <a class="nav-link" href="plugins/plugins.html">Plugins</a> <ul class="nav-list"> <li class="nav-item" data-depth="2"> <button class="nav-item-toggle"></button> <a class="nav-link" href="plugins/transforms.html">Transform Plugins</a> <ul class="nav-list"> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/abort.html">Abort</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/checksum.html">Add a checksum</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/constant.html">Add Constants</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/addsequence.html">Add Sequence</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/fieldschangesequence.html">Add value fields changing sequence</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/addxml.html">Add XML</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/analyticquery.html">Analytic Query</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/append.html">Append Streams</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beambigqueryinput.html">Beam BigQuery Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beambigqueryoutput.html">Beam BigQuery Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beampublisher.html">Beam GCP Pub/Sub : Publish</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beamsubscriber.html">Beam GCP Pub/Sub : Subscribe</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beaminput.html">Beam Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beamconsume.html">Beam Kafka Consume</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beamproduce.html">Beam Kafka Produce</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beamoutput.html">Beam Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beamtimestamp.html">Beam Timestamp</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/beamwindow.html">Beam Window</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/blockingtransform.html">Blocking transform</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/blockuntiltransformsfinish.html">Blocking until transforms finish</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/calculator.html">Calculator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/dbproc.html">Call DB procedure</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/changefileencoding.html">Change file encoding</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/filelocked.html">Check if file is locked</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/webserviceavailable.html">Check if webservice is available</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/clonerow.html">Clone row</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/closure.html">Closure</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/columnexists.html">Column exists</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/combinationlookup.html">Combination lookup/update</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/concatfields.html">Concat Fields</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/rowstoresult.html">Copy rows to result</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/creditcardvalidator.html">Credit card validator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/csvinput.html">CSV File Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/cubeinput.html">Cube input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/cubeoutput.html">Cube output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/databasejoin.html">Database Join</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/databaselookup.html">Database Lookup</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/datagrid.html">Data Grid</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/validator.html">Data Validator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/delay.html">Delay row</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/delete.html">Delete</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/detectemptystream.html">Detect Empty Stream</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/dimensionlookup.html">Dimension lookup/update</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/dummy.html">Dummy (do nothing)</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/dynamicsqlrow.html">Dynamic SQL row</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/edi2xml.html">Edi to XML</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/mailinput.html">Email Messages Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/excelwriter.html">Excel writer</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/execprocess.html">Execute a process</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/execsqlrow.html">Execute row SQL script</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/sql.html">Execute SQL script</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/fake.html">Fake data</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/fileexists.html">File exists</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/filesfromresult.html">Files from result</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/filestoresult.html">Files to result</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/filterrows.html">Filter Rows</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/fuzzymatch.html">Fuzzy match</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/randomvalue.html">Generate Random Value</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/getxmldata.html">Get Data From XML</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/getfilenames.html">Get filenames</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/getfilesrowcount.html">Get files rowcount</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/getslavesequence.html">Get ID from slave server</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/recordsfromstream.html">Get records from stream </a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/rowsfromresult.html">Get Rows from Result</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/getsubfolders.html">Get SubFolder names</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/systemdata.html">Get System Info</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/gettablenames.html">Get table names</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/getvariable.html">Get variables</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/groupby.html">Group By</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/http.html">HTTP client</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/httppost.html">HTTP Post</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/detectlastrow.html">Identify last row in a stream</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/ifnull.html">If Null</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/injector.html">Injector</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/insertupdate.html">Insert / Update</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/javafilter.html">Java Filter</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/javascript.html">JavaScript</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/joinrows.html">Join Rows</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/jsoninput.html">JSON Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/jsonoutput.html">JSON Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/ldapinput.html">LDAP Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/ldapoutput.html">LDAP Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/ldifinput.html">LDIF Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/loadfileinput.html">Load file content in memory</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/mail.html">Mail</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/mailvalidator.html">Mail Validator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/memgroupby.html">Memory Group By</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/mergejoin.html">Merge Join</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/mergerows.html">Merge rows (diff)</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/metainject.html">Metadata Injection</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/excelinput.html">Microsoft Excel input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/exceloutput.html">Microsoft Excel output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/mongodbinput.html">MongoDB Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/mongodboutput.html">MongoDB Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/multimerge.html">Multiway Merge Join</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/nullif.html">Null If</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/numberrange.html">Number range</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/pgpdecryptstream.html">PGP decrypt stream</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/pgpencryptstream.html">PGP encrypt stream</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/pipelineexcecutor.html">Pipeline Executor</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/pgbulkloader.html">PostgreSQL Bulk Loader</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/processfiles.html">Process files</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/propertyinput.html">Read data (key, value) from properties files.</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/regexeval.html">Regex Evaluation</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/replacestring.html">Replace in String</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/reservoirsampling.html">Reservoir Sampling</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/rest.html">REST Client</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/denormaliser.html">Row Denormaliser</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/flattener.html">Row Flattener</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/rowgenerator.html">Row Generator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/normaliser.html">Row Normaliser</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/ssh.html">Run SSH commands</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/selectvalues.html">Select Values</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/syslog.html">Send Message to Syslog</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/setvaluefield.html">Set field Value</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/setvalueconstant.html">Set field value to a constant</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/setvariable.html">Set Variables</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/sortedmerge.html">Sorted Merge</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/sort.html">Sort Rows</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/splitfieldtorows.html">Split fields</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/fieldsplitter.html">Split Fields</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/sqlfileoutput.html">SQL File Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/streamlookup.html">Stream Lookup</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/streamschemamerge.html">Stream Schema Merge</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/stringoperations.html">String operations</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/stringcut.html">Strings cut</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/switchcase.html">Switch / Case</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/synchronizeaftermerge.html">Synchronize after merge </a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/tablecompare.html">Table Compare</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/tableexists.html">Table Exists</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/tableinput.html">Table Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/tableoutput.html">Table Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/terafast.html">Teradata Bulk Loader</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/textfileinput.html">Text File Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/textfileoutput.html">Text File Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/tokenreplacement.html">Token Replacement</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/uniquerows.html">Unique Rows</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/uniquerowsbyhashset.html">Unique Rows (HashSet)</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/update.html">Update</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/userdefinedjavaclass.html">User Defined Java Class</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/janino.html">User Defined Java Expression</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/valuemapper.html">Value Mapper</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/webservices.html">Web services lookup</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/workflowexecutor.html">Workflow Executor</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/propertyoutput.html">Write data to properties file</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/writetolog.html">Write to log</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/xmlinputstream.html">XML Input Stream (StAX) </a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/xmljoin.html">XML Join</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/xmloutput.html">XML Output</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/xsdvalidator.html">XSD Validator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/xslt.html">XSL Transformation</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/yamlinput.html">Yaml Input</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/transforms/zipfile.html">Zip file</a> </li> </ul> </li> <li class="nav-item" data-depth="2"> <button class="nav-item-toggle"></button> <a class="nav-link" href="plugins/databases.html">Database Plugins</a> <ul class="nav-list"> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/derby.html">Apache Derby</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/as400.html">AS400</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/cache.html">Cache</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/db2.html">DB2</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/exasol.html">Exasol</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/firebird.html">firebird</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/googlebigquery.html">Google BigQuery</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/greenplum.html">Greenplum</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/sqlbase.html">Gupta SQLBase</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/h2.html">H2</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/hypersonic.html">Hypersonic</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/universe.html">IBM Universe</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/infinidb.html">InfiniDB</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/infobright.html">Infobright DB</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/informix.html">informix</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/ingres.html">ingres</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/interbase.html">interbase</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/kingbasees.html">kingbase es</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/mariadb.html">Mariadb</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/sapdb.html">MaxDB (SAP DB)</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/monetdb.html">monetdb</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/mssql.html">MS SqlServer</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/mssqlnative.html">MS SqlServer (Native)</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/mysql.html">MySql</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/netezza.html">Netezza</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/oracle.html">Oracle</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/oraclerdb.html">Oracle RDB</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/postgresql.html">PostgreSQL</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/redshift.html">Redshift</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/snowflake.html">Snowflake</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/sqlite.html">SqLite</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/sybase.html">Sybase</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/sybaseiq.html">Sybase IQ</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/teradata.html">Teradata</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/vectorwise.html">Vectorwise</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/databases/vertica.html">Vertica</a> </li> </ul> </li> <li class="nav-item" data-depth="2"> <button class="nav-item-toggle"></button> <a class="nav-link" href="plugins/actions.html">Action Plugins</a> <ul class="nav-list"> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/abort.html">Abort</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/addresultfilenames.html">Add filenames to result</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/checkdbconnection.html">Check Db connections</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/checkfilelocked.html">Check Files Locked</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/webserviceavailable.html">Check if Webservice is available</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/xmlwellformed.html">Check if XML file is well formed</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/filesexist.html">Checks if files exists</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/columnsexist.html">Columns exist in a table</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/copyfiles.html">Copy Files</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/copymoveresultfilenames.html">Copy Or Move Files from result</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/createfile.html">Create File</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/createfolder.html">Create Folder</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/pgpdecryptfiles.html">Decrypt files with PGP</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/delay.html">Delay</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/deletefile.html">Delete File</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/deleteresultfilenames.html">Delete filenames from result</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/deletefiles.html">Delete Files</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/deletefolders.html">Delete Folders</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/msgboxinfo.html">Display Msgbox info</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/dostounix.html">Dos To Unix</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/dtdvalidator.html">DTD Validator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/dummy.html">Dummy</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/pgpencryptfiles.html">Encrypt files with PGP</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/repeat-end.html">End Repeat</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/evalfilesmetrics.html">Evaluate File Metrics</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/evaluatetablecontent.html">Evaluate rows number in a table</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/filecompare.html">File compare</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/fileexists.html">File Exists</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/folderisempty.html">Folder is Empty</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/folderscompare.html">Folders Compare</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/ftpdelete.html">FTP delete</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/ftp.html">Get a file with FTP</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/ftpsget.html">Get a file with FTPS</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/sftp.html">Get a file with SFTP</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/getpop.html">Get Mail from POP</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/http.html">Http Get</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/eval.html">JavaScript</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/mail.html">Mail</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/mailvalidator.html">Mail Validator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/movefiles.html">Move Files</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/mssqlbulkload.html">MS SQL Server Bulk Loader</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/mysqlbulkfile.html">MySQL Bulk File</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/mysqlbulkload.html">MySQL Bulk Loader</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/pgpfiles.html">PGP Files</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/ping.html">Ping</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/pipeline.html">Pipeline</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/ftpput.html">Put a file with FTP</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/sftpput.html">Put a file with SFTP</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/repeat.html">Repeat</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/syslog.html">Send information using Syslog</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/sendnagiospassivecheck.html">Send Nagios check</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/setvariables.html">Set Variables</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/shell.html">Shell Action</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/simpleeval.html">Simple Evaluation</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/snmptrap.html">Snmp Trap</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/sql.html">SQL Script Executor</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/start.html">Start</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/success.html">Success Action</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/tableexists.html">Table Exists</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/telnet.html">Telnet a host</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/truncatetables.html">Truncate Tables</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/unzip.html">Unzip</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/ftpsput.html">Upload files to FTPS</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/pgpverify.html">Verify file signature with PGP</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/waitforfile.html">Wait for file</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/waitforsql.html">Wait for SQL</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/workflow.html">Workflow</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/writetofile.html">Write to File</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/writetolog.html">Write to Log</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/xsdvalidator.html">XSD Validator</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/xslt.html">XSL Transformation</a> </li> <li class="nav-item" data-depth="3"> <a class="nav-link" href="plugins/actions/zipfile.html">Zip File</a> </li> </ul> </li> </ul> </li> <li class="nav-item" data-depth="1"> <button class="nav-item-toggle"></button> <a class="nav-link" href="samples/samples.html">Samples</a> <ul class="nav-list"> <li class="nav-item" data-depth="2"> <a class="nav-link" href="samples/dummy-sample.html">Dummy Sample</a> </li> </ul> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="faq/faq.html">Frequently Asked Questions</a> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="image-testpage.html">Image testing page</a> </li> </ul> </li> </ul> </nav> </div> <div class="nav-panel-explore" data-panel="explore"> <div class="context"> <span class="title">User manual</span> <span class="version">latest</span> </div> <ul class="components"> <li class="component"> <span class="title">Development Documentation</span> <ul class="versions"> <li class="version is-latest"> <a href="../../dev-manual/latest/index.html">latest</a> </li> </ul> </li> <li class="component"> <span class="title">Technical Documentation</span> <ul class="versions"> <li class="version is-latest"> <a href="../../tech-manual/latest/index.html">latest</a> </li> </ul> </li> <li class="component is-current"> <span class="title">User manual</span> <ul class="versions"> <li class="version is-current is-latest"> <a href="index.html">latest</a> </li> </ul> </li> </ul> </div> </div> </aside> </div> <main> <div class="toolbar" role="navigation"> <button class="nav-toggle"></button> <a href="index.html" class="home-link"></a> <nav class="breadcrumbs" aria-label="breadcrumbs"> <ul> <li><a href="index.html">User manual</a></li> <li><a href="getting-started.html">Getting Started</a></li> </ul> </nav> <div class="edit-this-page"><a href="https://github.com/apache/incubator-hop-docs/edit/asf-site/hop-user-manual/modules/ROOT/pages/getting-started.adoc">Edit this Page</a></div> </div> <article class="doc"> <h1 class="page">Getting Started</h1> <div class="sect1"> <h2 id="_getting_started_with_hop"><a class="anchor" href="#_getting_started_with_hop"></a>Getting Started with Hop</h2> <div class="sectionbody"> <div class="ulist"> <ul> <li> <p><a href="https://hop.apache.org/download/">Download</a> a recent Hop build.</p> </li> <li> <p>unzip hop to a local directory</p> </li> <li> <p>change to the hop directory</p> </li> </ul> </div> </div> </div> <div class="sect1"> <h2 id="_introducing_hop"><a class="anchor" href="#_introducing_hop"></a>Introducing Hop</h2> <div class="sectionbody"> <div class="paragraph"> <p>Hop is a metadata driven environment where you manage your data processing workflows.</p> </div> <div class="paragraph"> <p>Before anything else, we need to explain Hop’s two main concepts:</p> </div> <div class="ulist"> <ul> <li> <p><strong>Workflow</strong> is a (by default) sequential process that has a starting point and one or more endpoints. Between the start and endpoint, a variety of 'actions' can be performed. These actions can range from executing other workflows or pipelines, archiving files that were processed, sending error messages or success notifications and much more.</p> </li> <li> <p><strong>Pipelines</strong> are more granular items of work. A pipeline is where the actual work is done. Pipelines consist of a chain of transforms that read, process or write data. Depending on the execution engine your pipelines run, this can be in batch, streaming or a hybrid mode.</p> </li> </ul> </div> <div class="paragraph"> <p>The actions in a workflow and the transforms in a pipeline are connected by 'hops'. Hop are visual links between actions (workflows) and transforms (pipelines).</p> </div> <div class="paragraph"> <p>As you’ll discover soon, the process of creating workflows and pipelines is very similar.</p> </div> <div class="paragraph"> <p>However, there are a number of conceptual differences between workflows and pipelines that you have to keep in mind:</p> </div> <div class="ulist"> <ul> <li> <p>the pipeline engine executes all transforms in a pipeline simultaneously and in parallel. The workflow engine executes all actions in a workflow sequentially by default. When action finishes, the workflow engine checks which action needs to be executed next.</p> </li> <li> <p>hops in a pipeline pass data between transforms. In a workflow, hops can conditionally determine which action the workflow needs to execute next (on success, on failure, unconditionally)</p> </li> <li> <p>because of their sequential nature, workflows have 1 action to start from and 1 or more end actions. Pipelines can start with input from multiple transforms simultaneously.</p> </li> </ul> </div> <div class="paragraph"> <p>The following tools are at your disposal to work with Hop flows and pipelines:</p> </div> <div class="ulist"> <ul> <li> <p>the <strong>Hop Gui</strong> is your visual IDE to build, preview, run, test, deploy, …​ workflows and pipelines.</p> </li> <li> <p>the <strong>Hop Server</strong> is a lightweight web server that provides a REST api to run workflows and pipelines remotely.</p> </li> <li> <p><strong>Hop Run</strong> is a command line utility to run workflows and pipelines.</p> </li> </ul> </div> </div> </div> <div class="sect1"> <h2 id="_the_hop_gui"><a class="anchor" href="#_the_hop_gui"></a>The Hop GUI</h2> <div class="sectionbody"> <div class="paragraph"> <p>The Hop Gui is your local development environment to build, run, preview and debug (work)flows and pipelines.</p> </div> <div class="paragraph"> <p>Check out this short video to learn how to download, unzip and start the Hop Gui (on Windows).</p> </div> <div class="paragraph"> <p><!-- [html-validate-disable-next deprecated] -→ video::RMIOTmZK-YE[youtube, width=75%, height=400px]</p> </div> <div class="sect2"> <h3 id="_start_the_hop_gui"><a class="anchor" href="#_start_the_hop_gui"></a>Start the Hop GUI</h3> <div class="paragraph"> <p>On Linux:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash"> ./hop-gui.sh</code></pre> </div> </div> <div class="paragraph"> <p>On Windows:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash">hop-gui.bat</code></pre> </div> </div> </div> <div class="sect2"> <h3 id="_hop_gui_walkthrough"><a class="anchor" href="#_hop_gui_walkthrough"></a>Hop GUI Walkthrough</h3> <div class="paragraph"> <p>After starting the Hop Gui, you’ll be presented with a window like the one below.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-hop-gui.png" alt="Hop Gui" width="75%"> </div> </div> <div class="paragraph"> <p>After clicking the 'New' icon in the upper left corner, you’ll be presented with the window below. Choose either 'New Workflow' or 'New Pipeline'.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-new-dialog.png" alt="Hop - New Dialog" width="75%"> </div> </div> <div class="sect3"> <h4 id="_pipeline_editor_overview"><a class="anchor" href="#_pipeline_editor_overview"></a>Pipeline Editor Overview</h4> <div class="paragraph"> <p>Your new pipeline is created, and you’ll see the dialog below.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-new-pipeline.png" alt="Hop - New Pipeline" width="75%"> </div> </div> <div class="paragraph"> <p>Let’s walk through the top toolbar:</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-pipeline-toolbar.png" alt="Hop - Pipeline Toolbar"> </div> </div> <div class="table-wrapper"><table class="tableblock frame-all grid-all"> <colgroup> <col> <col> <col> </colgroup> <thead> <tr> <th class="tableblock halign-left valign-top">Action</th> <th class="tableblock halign-left valign-top">Icon</th> <th class="tableblock halign-left valign-top">Description</th> </tr> </thead> <tbody> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">run</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/run.svg" alt="Run" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">start the execution of the pipeline</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">pause</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/pause.svg" alt="Pause" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">pause the execution of the pipeline</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">stop</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/stop.svg" alt="Stop" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">stop the execution of the pipeline</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">preview</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/view.svg" alt="Preview" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">preview the pipeline</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">debug</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/debug.svg" alt="Debug" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">debug the pipeline</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">print</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/print.png" alt="Print" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">print the pipeline</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">undo</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/Antu_edit-undo.svg" alt="Undo" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">undo an operation</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">redo</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/Antu_edit-redo.svg" alt="Redo" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">redo an operation</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">align</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/snap-to-grid.svg" alt="Snap To Grid" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">align the specified (selected) transforms to the specified grid size</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">align left</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/align-left.svg" alt="Align Left" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">align the selected transforms with left-most selected transform in the selection</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">align right</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/align-right.svg" alt="Align Right" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">align the selected transforms with right-most selected transform in the selection</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">align top</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/align-top.svg" alt="Align Top" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">align the selected transforms with top-most selected transform in the selection</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">align bottom</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/align-bottom.svg" alt="Align Bottom" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">align the selected transforms with bottom-most selected transform in the selection</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td> <td class="tableblock halign-left valign-top"></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">distribute horizontally</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/distribute-horizontally.svg" alt="Distribute Horizontally" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">Distribute the selected transforms evenly between the left-most and right-most transform in your selection</p></td> </tr> <tr> <td class="tableblock halign-left valign-top"><p class="tableblock">distribute vertically</p></td> <td class="tableblock halign-left valign-top"><p class="tableblock"><span class="image"><img src="_images/getting-started/icons/distribute-vertically.svg" alt="Distribute Vertically" width="25px"></span></p></td> <td class="tableblock halign-left valign-top"><p class="tableblock">Distribute the selected transforms evenly between the top-most and bottom-most transform in your selection</p></td> </tr> </tbody> </table></div> </div> </div> <div class="sect2"> <h3 id="_build_your_first_pipeline"><a class="anchor" href="#_build_your_first_pipeline"></a>Build Your First Pipeline</h3> <div class="sect3"> <h4 id="_concepts"><a class="anchor" href="#_concepts"></a>Concepts</h4> <div class="paragraph"> <p>Pipelines consist of two main work items:</p> </div> <div class="ulist"> <ul> <li> <p><strong>transforms</strong> are the basic operations in your pipeline. A pipeline typically consists of a lot of transforms that are chained together by hops. Transforms are granular, in the sense that each transform is designed and optimized to perform one and only one task. Although one transform by itself may not offer spectacular functionality, the combination of all transforms in a pipeline is makes your pipelines powerful.</p> </li> <li> <p><strong>hops</strong> link transforms together. When a transform finishes processing the data set it received, that data set is passed to the next transform through a hop. Hops are uni-directional (data can’t flow backwards). Hops only buffer and pass data around, the hop itself is transform-agnostic, it doesn’t know anything about the transforms it passes data from or to. Some transforms can read from or write to other transforms conditionally to or from a number of other transforms, but this a transform-specific configuration. The hop is unaware of it. Hops can be disabled by clicking on them, or through right-click → disable.</p> </li> </ul> </div> </div> <div class="sect3"> <h4 id="_add_transforms"><a class="anchor" href="#_add_transforms"></a>Add Transforms</h4> <div class="paragraph"> <p>Click anywhere in the pipeline canvas, the area where you’ll see the image below.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-click-anywhere.png" alt="Hop - Click Anywhere" width="45%"> </div> </div> <div class="paragraph"> <p>Upon clicking, you’ll be presented with the dialog shown below. The search box at the top of this dialog works for transform, name, tags (TODO) etc. Once you’ve found the transform you’re looking for, click on it to add it to your pipeline. An alternative to clicking is arrow key navigation + enter. Repeat this step now or whenever you want to add more transforms to your pipeline. Once you’ve added a transform to your pipeline, you can drag to reposition it.</p> </div> <div class="paragraph"> <p>TODO: link to transform documentation.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-add-transform.png" alt="Hop - Add Transform" width="75%"> </div> </div> <div class="paragraph"> <p>Add a 'Generate Rows' and a 'Add Sequence' transform, and your pipeline should like the one below.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-add-two-transforms.png" alt="Hop - Add two transforms" width="75%"> </div> </div> </div> <div class="sect3"> <h4 id="_add_a_hop"><a class="anchor" href="#_add_a_hop"></a>Add a Hop</h4> <div class="paragraph"> <p>There are a number of ways to create a hop:</p> </div> <div class="ulist"> <ul> <li> <p>shift-drag: while holding down the shift key on your keyboard. Click on a transform, while holding down your primary mouse button, drag to the second transform. Release the primary mouse button and the shift key.</p> </li> <li> <p>scroll-drag: scroll-click on a transform , while holding down your mouse’s scroll button, drag to the second transform. Release the scroll button.</p> </li> <li> <p>click on a transform in your pipeline to open the 'click anywhere' dialog. Click the 'Create hop' image::getting-started/icons/HOP.svg[Create hop, 25px, align="bottom"] button and select the transform you want to create the hop to.</p> </li> </ul> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-create-hop.png" alt="Hop - Create Hop" width="65%"> </div> </div> </div> </div> <div class="sect2"> <h3 id="_run_your_first_pipeline"><a class="anchor" href="#_run_your_first_pipeline"></a>Run your first Pipeline</h3> <div class="paragraph"> <p>Click the 'run' button image::getting-started/icons/run.svg[Run, 25px, align="bottom"] in your pipeline toolbar</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-run-pipeline-dialog.png" alt="Hop - Create Hop" width="75%"> </div> </div> <div class="paragraph"> <p>Let’s walk through the options in this dialog</p> </div> <div class="ulist"> <ul> <li> <p>Pipeline run configurations, edit, new, manage your run configurations. Run configurations are used to specify a name, description and engine to run your pipeline.</p> </li> <li> <p>Log level: choose the log level for your pipeline. The available options are</p> <div class="ulist"> <ul> <li> <p>Nothing</p> </li> <li> <p>Error</p> </li> <li> <p>Minimal</p> </li> <li> <p>Basic (default)</p> </li> <li> <p>Detailed</p> </li> <li> <p>Debugging</p> </li> <li> <p>Row Level (very detailed)</p> </li> </ul> </div> </li> <li> <p>Clear log before running (enabled by default): logging information from previous runs will be cleared from the logging tab.</p> </li> <li> <p>parameters: This table will show the parameter name, default value and description. enter your runtime parameters in the 'value' field.</p> </li> <li> <p>variables: add the variable name and value you want to set in this tab.</p> </li> <li> <p>always show dialog on run (enabled by default): You’ll be presented with this dialog every time you run this dialog. When disabled, the pipeline will run with the default options.</p> </li> </ul> </div> <div class="paragraph"> <p>Click the 'New' button right next to the 'Pipeline run configuration'. Give your run configuration a name and (optionally) a description. Choose the 'local pipeline engine'. As the name implies, the 'local single threaded pipeline engine' runs the pipeline in a single CPU thread. The default 'local pipeline engine' will create a separate CPU thread for each transform in your pipeline to evenly spread the load of your pipeline over your CPU cores.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-run-configuration-dialog.png" alt="Hop - Run Configuration Dialog" width="75%"> </div> </div> <div class="paragraph"> <p>Click 'Ok' to create your configuration and select it from the dropdown list. For this getting started guide, we’ll leave all other options to the defaults. Click 'Launch'.</p> </div> <div class="paragraph"> <p>Since we haven’t saved our pipeline yet, you’ll be prompted to do so by the dialog below.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-save-pipeline.png" alt="Hop - Save Pipeline" width="55%"> </div> </div> <div class="paragraph"> <p>Your pipeline will finish in a matter of milliseconds, and the 'Execution Result' view will show up at the bottom of your IDE. This view has 5 tabs:</p> </div> <div class="ulist"> <ul> <li> <p>transform metrics: transformName, read, written, input, output, update, rejected, errors, buffers input, buffers output, speed, status (TODO: elaborate)</p> </li> <li> <p>logging: the logging output for your pipeline</p> </li> <li> <p>preview data: a preview of the data for the selected transform. This grid shows the data as it passed through this transform.</p> </li> <li> <p>metrics: TODO</p> </li> <li> <p>performance graph: TODO</p> </li> </ul> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-execution-results-metrics.png" alt="Hop - Execution Results Metrics" width="75%"> </div> </div> </div> <div class="sect2"> <h3 id="_preview_your_first_pipeline"><a class="anchor" href="#_preview_your_first_pipeline"></a>Preview your first Pipeline</h3> <div class="paragraph"> <p>While developing your pipeline, you’ll often want to check your data as it enters or exits a transform. Previews are an easy way to take a glance at the state of your data stream as it exits a transform.</p> </div> <div class="paragraph"> <p>To preview the data that is processed by a transform, click on a transform and select 'Preview output'. The same result can be achieved by selecting a transform in your pipeline (rectangle select) and clicking the preview (eye) icon in the pipeline toolbar.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-preview-pipeline.png" alt="Hop - Preview Transform" width="75%"> </div> </div> <div class="paragraph"> <p>You’ll be presented with the dialog below. You can change the number of rows to preview (1000 by default), but in most cases, you’ll just want to hit the green 'Quick Launch' button.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-preview-dialog.png" alt="Hop - Preview Dialog" width="75%"> </div> </div> <div class="paragraph"> <p>Once your pipeline finished processing the selected number of rows for the selected transform, a new popup dialog will show your preview results.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-preview-results.png" alt="Hop - Preview Results" width="75%"> </div> </div> <div class="admonitionblock important"> <div class="table-wrapper"><table> <tr> <td class="icon"> <i class="fa icon-important" title="Important"></i> </td> <td class="content"> your <strong>entire</strong> pipeline is executed for a preview, you’re just taking a peek into the processing at the selected transform. If your pipeline modifies data (writes, updates, deletes) further down the stream, those actions <strong>*will</strong> be performed, even if you’re previewing an earlier transform. </td> </tr> </table></div> </div> <div class="paragraph"> <p>Let’s take a quick look at the buttons at the bottom of this dialog:</p> </div> <div class="ulist"> <ul> <li> <p>Close: closes the preview dialog. The pipeline will remain paused, and will therefore still be active.</p> </li> <li> <p>Stop: stop the preview and the pipeline execution.</p> </li> <li> <p>Get more rows: fetch the next 1000 (or any other selected amount of) rows for preview.</p> </li> </ul> </div> </div> <div class="sect2"> <h3 id="_debug_your_first_pipeline"><a class="anchor" href="#_debug_your_first_pipeline"></a>Debug your first Pipeline</h3> <div class="paragraph"> <p>Debugging a pipeline’s transform is very similar to previewing. Instead of pausing the pipeline execution after a given number of rows, the pipeline is paused when a given condition is met. The process to start a debug session is similar to starting the preview: click on a transform and select 'Debug output' from the pop-up dialog, or select a transform and hit the bug-icon in the pipeline toolbar.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-debug-pipeline.png" alt="Hop - Preview Transform" width="75%"> </div> </div> <div class="paragraph"> <p>You’ll be presented with the dialog below. You’ll recognize this dialog from the 'preview' we just did, but instead, the 'Retrieve first rows (preview)' option is now unchecked, and 'Pause pipeline on condition' is checked.</p> </div> <div class="paragraph"> <p>In the 'Break-point / pause condition' below that option, you can specify on which condition you want to debug. This dialog is the same as the <a href="https://hop.apache.org/manual/latest/plugins/transforms/filterrows.html">Filter Rows</a> transform.</p> </div> <div class="paragraph"> <p>In our very basic example, we’ve set a breakpoint on 'valuename > 5'.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-debug-dialog.png" alt="Hop - Preview Dialog" width="75%"> </div> </div> <div class="paragraph"> <p>With the 'valuename > 5' breakpoint, our pipeline is paused as soon as this condition is met (valuename = 6). The rows preceding that moment are also shown, so you can investigate how your data was processed before the breakpoint condition was true.</p> </div> <div class="paragraph"> <p>Similar to the preview options, you can close, stop or continue the debugging ('Get more rows'). When you tell your pipeline to 'Get more rows', the pipeline execution will be resumed until the breakpoint condition is met once more, instead of just fetching the next 1000 (default) rows.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-debug-results.png" alt="Hop - Preview Results" width="75%"> </div> </div> </div> <div class="sect2"> <h3 id="_create_your_first_workflow"><a class="anchor" href="#_create_your_first_workflow"></a>Create your first Workflow</h3> <div class="paragraph"> <p>The design and execution of workflows is very similar to that of pipelines. However, keep in mind that there are significant differences between how Hop handles workflows and pipelines under the hood.</p> </div> <div class="paragraph"> <p>To create a workflow, hit the 'new' icon or 'CTRL-N'. From the pop-up dialog, select 'New workflow'.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-new-workflow.png" alt="Hop - New Workflow" width="75%"> </div> </div> <div class="paragraph"> <p>Add the following actions to your workflow and create the hops to connect them:</p> </div> <div class="ulist"> <ul> <li> <p>Start</p> </li> <li> <p>Pipeline</p> </li> <li> <p>Success</p> </li> </ul> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-new-workflow-actions.png" alt="Hop - New Workflow with actions" width="75%"> </div> </div> <div class="paragraph"> <p>Double-click or single-click and choose 'Edit action' to configure the action for the pipeline you just created.</p> </div> <div class="paragraph"> <p>In the pipeline dialog, use the 'Browse' button to select your pipeline and give the action an appropriate name, for example 'First Pipeline'.</p> </div> <div class="paragraph"> <p>Click 'OK'.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-new-workflow-pipeline-action.png" alt="Hop - New Workflow pipeline action" width="75%"> </div> </div> <div class="paragraph"> <p>Notice how the hops in your workflow are a little different from what you’ve seen in pipeline hops.</p> </div> <div class="paragraph"> <p>Add a fourth action 'Abort' and create a hop from your pipeline action.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-new-workflow-abort.png" alt="Hop - New Workflow abort" width="75%"> </div> </div> <div class="paragraph"> <p>You now have the three types of hops that are available in workflows:</p> </div> <div class="ulist"> <ul> <li> <p>unconditional (lock icon, black hop): 'unconditional' hops are followed no matter what the exit code (true/false) of the previous action is</p> </li> <li> <p>success (green hop, check mark): 'success' hops are used when the previous action executed successfully.</p> </li> <li> <p>failure (red hop, error mark): 'failure' or 'error' hops are followed when the previous action failed.</p> </li> </ul> </div> <div class="admonitionblock note"> <div class="table-wrapper"><table> <tr> <td class="icon"> <i class="fa icon-note" title="Note"></i> </td> <td class="content"> The hop type can be changed by clicking on the hop’s icon. </td> </tr> </table></div> </div> <div class="paragraph"> <p>With these three hop types and the actions at your disposal, you’re ready to create powerful data orchestration workflows.</p> </div> </div> <div class="sect2"> <h3 id="_run_your_first_workflow"><a class="anchor" href="#_run_your_first_workflow"></a>Run your first Workflow</h3> <div class="paragraph"> <p>As with designing workflows, the steps to run a workflow are very similar to running a pipeline.</p> </div> <div class="paragraph"> <p>Click the 'run' button <span class="image"><img src="_images/getting-started/icons/run.svg" alt="Run" width="25px"></span> in your workflow toolbar</p> </div> <div class="paragraph"> <p>In the workflow run dialog, hit the 'New' button in the upper right corner to create a new 'Workflow run configuration'.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-run-workflow-dialog.png" alt="Hop - New Workflow Config" width="75%"> </div> </div> <div class="paragraph"> <p>In the dialog that pops up, add 'Local Workflow' as the workflow configuration name and choose the 'Local workflow engine'.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-run-workflow-config-dialog.png" alt="Hop - New Workflow Config Dialog" width="75%"> </div> </div> <div class="paragraph"> <p>Click 'OK' to return to the workflow run dialog, make sure your configuration is selected and hit 'Launch'.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-run-workflow-with-config.png" alt="Hop - New Workflow With Config Dialog" width="75%"> </div> </div> <div class="paragraph"> <p>This workflow with our very basic pipeline should execute in less than one second. You’ll now have the execution results pane which again looks very similar to the pipeline execution results.</p> </div> <div class="paragraph"> <p>The first tab in your workflow execution is 'Logging'. This tab shows the logging information for your entire workflow. Any errors that occurred in your workflow will be highlighted in red.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-run-workflow-logging.png" alt="Hop - New Workflow Logging" width="75%"> </div> </div> <div class="paragraph"> <p>The second tab are your workflow metrics. This tab is less verbose, but gives you an action-by-action overview of the execution of your workflow. The black, green and red color codings indicate information, success and failure. In larger worfklows, the metrics tab will give you a quick overview of what happened in your workflow, what the required time per action was, etc.</p> </div> <div class="paragraph"> <p>You’ll use the logging tab to find more detailed information about what happened in your workflow or in a particular action.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-run-workflow-metrics.png" alt="Hop - New Workflow Metrics" width="75%"> </div> </div> </div> </div> </div> <div class="sect1"> <h2 id="_hop_server"><a class="anchor" href="#_hop_server"></a>Hop Server</h2> <div class="sectionbody"> <div class="paragraph"> <p>After you’ve designed and tested your pipeline or transform locally, you may want to run it on a headless machine.</p> </div> <div class="paragraph"> <p>The Hop Server is a light weight web server that you can use to run your workflows and pipelines remotely.</p> </div> <div class="paragraph"> <p>First, we’ll have to start the server. Head over to your Hop directory, and locate the 'hop-server' scripts (sh for Mac/Linux, bat for Windows).</p> </div> <div class="paragraph"> <p>Running the script without any arguments will print its usage:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash">Usage: hop-server <Interface address> <Port> [-h] [-p <arg>] [-s] [-u <arg>] |
| or |
| Usage: hop-server <Configuration File> |
| Starts or stops the hopServer server. |
| -h,--help This help text |
| -p,--password <arg> The administrator password. Required only if |
| stopping the Hop Server server. |
| -s,--stop Stop the running hopServer server. This is only |
| allowed when using the hostname/port form of the |
| command. |
| -u,--userName <arg> The administrator user name. Required only if |
| stopping the Hop Server server. |
| Example: hop-server.sh 127.0.0.1 8080 |
| Example: hop-server.sh 192.168.1.221 8081 |
| Example: hop-server.sh /foo/bar/hop-server-config.xml |
| Example: hop-server.sh http://www.example.com/hop-server-config.xml |
| Example: hop-server.sh 127.0.0.1 8080 -s -u cluster -p cluster</code></pre> </div> </div> <div class="paragraph"> <p>As an example, let’s run our server on our local machine on port 8085:</p> </div> <div class="paragraph"> <p>On Linux:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash"> ./hop-server.sh localhost 8085</code></pre> </div> </div> <div class="paragraph"> <p>On Windows:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash">hop-server.bat localhost 8085</code></pre> </div> </div> <div class="paragraph"> <p>The startup process shouldn’t take more than 1 or 2 seconds, and should show 2 lines of logging information:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash">2020/04/30 16:22:55 - HopServer - Installing timer to purge stale objects after 1440 minutes. |
| 2020/04/30 16:22:55 - HopServer - Created listener for webserver @ address : localhost:8085</code></pre> </div> </div> <div class="paragraph"> <p>In your favorite browser, go to <a href="http://localhost:8085" class="bare">http://localhost:8085</a> and sign in with the default user 'cluster' and password 'cluster'.</p> </div> <div class="paragraph"> <p>Click the 'show status' link below to get to page shown in the second screenshot.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-server-index.png" alt="Hop - Server Index" width="75%"> </div> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-server-status.png" alt="Hop - Server Status" width="75%"> </div> </div> <div class="paragraph"> <p>We now have verified our server is up and running. Let’s return to Hop Gui to configure a run configuration for it. Click the 'New' icon or 'CTRL-N' and select 'Slave Server'.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-new-slave.png" alt="Hop - New Slave" width="75%"> </div> </div> <div class="paragraph"> <p>In the slave server dialog, enter the details for the local server we just created.</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-new-slave-config.png" alt="Hop - New Slave Config" width="75%"> </div> </div> <div class="paragraph"> <p>With our slave server in place, all that’s left to do is to create a run configuration for this server. Head back to your pipeline (again, the process is similar for workflows), and hit 'run'. Before running your pipeline, create a new 'Pipeline run configuration'.</p> </div> <div class="paragraph"> <p>Name this configuration 'Remote Pipeline', select 'Remote pipeline engine' as the engine type, select the 'local' run configuration we created earlier, and select 'localhost' for the slave server we just created.</p> </div> <div class="paragraph"> <p>Select this run configuration and run your pipeline. Your execution results will be almost identical to the locale execution you did earlier, however, the logs will show you executed the pipeline remotely:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash">2020/04/30 17:01:33 - first_pipeline - Executing this pipeline using the Remote Pipeline Engine with run configuration 'Remote Pipeline' |
| ... |
| ... |
| ... |
| 2020/04/30 17:01:34 - first_pipeline - Execution finished on a remote pipeline engine with run configuration 'Remote Pipeline'</code></pre> </div> </div> <div class="paragraph"> <p>The execution results for this pipeline will now be available in our server’s status page as well:</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-server-status-after-run.png" alt="Hop - Server Status" width="75%"> </div> </div> <div class="paragraph"> <p>Select the pipeline or workflow line that you want to investigate, and choose one of the options from the options in the upper left corner of the pipeline or workflow overview table. Click the eye icon to open the details for that specfific execution:</p> </div> <div class="imageblock text-left"> <div class="content"> <img src="_images/getting-started/getting-started-server-status-after-run-details.png" alt="Hop - Server Status Details" width="75%"> </div> </div> </div> </div> <div class="sect1"> <h2 id="_hop_run"><a class="anchor" href="#_hop_run"></a>Hop Run</h2> <div class="sectionbody"> <div class="paragraph"> <p>Hop Run is the last tool we’ll discuss in this getting started overview. In many cases, you’ll want to run your workflows and pipelines on a headless server, but don’t necessarily want to run through rest services or from Hop Gui.</p> </div> <div class="paragraph"> <p>Hop Run is a command line that can be used to run workflows or pipelines e.g. over ssh of from a cron job.</p> </div> <div class="paragraph"> <p>The command to run is 'hop-run' (sh on Mac/Linux, bat on Windows). Without any arguments, hop-run shows its usage syntax:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash">A filename is needed to run a workflow or pipeline |
| Usage: <main class> [-hotw] [-e=<environment>] [-f=<filename>] [-l=<level>] |
| [-r=<runConfigurationName>] [-p=<parameters>[, |
| <parameters>...]]... [-s=<systemProperties>[, |
| <systemProperties>...]]... |
| -e, --environment=<environment> |
| The name of the environment to use |
| -f, --file=<filename> The filename of the workflow or pipeline to run |
| -h, --help Displays this help message and quits. |
| -l, --level=<level> The debug level, one of NONE, MINIMAL, BASIC, DETAILED, |
| DEBUG, ROWLEVEL |
| -o, --printoptions Print the used options |
| -p, --parameters=<parameters>[,<parameters>...] |
| A comma separated list of PARAMETER=VALUE pairs |
| -r, --runconfig=<runConfigurationName> |
| The name of the Run Configuration to use |
| -s, --system-properties=<systemProperties>[,<systemProperties>...] |
| A comma separated list of KEY=VALUE pairs |
| -t, --pipeline Force execution of a pipeline |
| -w, --workflow Force execution of a workflow</code></pre> </div> </div> <div class="paragraph"> <p>Since we’ve been working with a very basic pipeline, running it from hop-run is as easy as specifying: * the pipeline filename to run * the run configuration to use</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash"> ./hop-run.sh -f /tmp/first_pipeline.hpl -r local</code></pre> </div> </div> <div class="paragraph"> <p>You’ll get output that will be very similar to the one below:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-bash hljs" data-lang="bash">2020/04/30 17:16:48 - first_pipeline - Executing this pipeline using the Local Pipeline Engine with run configuration 'local' |
| 2020/04/30 17:16:48 - first_pipeline - Execution started for pipeline [first_pipeline] |
| 2020/04/30 17:16:48 - Generate rows.0 - Finished processing (I=0, O=0, R=0, W=10, U=0, E=0) |
| 2020/04/30 17:16:48 - Add sequence.0 - Finished processing (I=0, O=0, R=10, W=10, U=0, E=0) |
| 2020/04/30 17:16:48 - first_pipeline - Pipeline duration : 0.079 seconds [ 0.079 ] |
| 2020/04/30 17:16:48 - first_pipeline - Execution finished on a local pipeline engine with run configuration 'local' |
| ./hop-run.sh -f /tmp/first_pipeline.hpl -r local 5.62s user 0.34s system 258% cpu 2.309 total</code></pre> </div> </div> </div> </div> <div class="sect1"> <h2 id="_where_to_go_from_here"><a class="anchor" href="#_where_to_go_from_here"></a>Where to go from here?</h2> <div class="sectionbody"> <div class="paragraph"> <p>We’ll be adding more documentation as we go, so keep an eye on the <a href="https://www.project-hop.org">Project Hop</a> documentation section.</p> </div> <div class="paragraph"> <p>A good place to start exploring is the detailed documentation for:</p> </div> <div class="ulist"> <ul> <li> <p><a href="https://hop.apache.org/manual/latest/plugins/actions.html">Workflow Actions</a></p> </li> <li> <p><a href="https://hop.apache.org/manual/latest/plugins/transforms.html">Pipeline Transforms</a></p> </li> </ul> </div> <div class="admonitionblock note"> <div class="table-wrapper"><table> <tr> <td class="icon"> <i class="fa icon-note" title="Note"></i> </td> <td class="content"> Project Hop considers high-quality documentation a very important part of the project. Help us to improve by creating tickets for any documentation errors, suggestions or feature requests in our <a href="https://jira.project-hop.org">JIRA system</a>. </td> </tr> </table></div> </div> </div> </div> </article> </main> </div> <footer> <div class="footer"> <dl> <dt>Overview</dt> <dd><a href="../../download/">Download</a></dd> <dd><a href="../../manual/latest/">Getting started</a></dd> </dl> <dl> <dt>Community</dt> <dd><a target="_blank" href="https://issues.apache.org/jira/projects/HOP/issues">Jira</a></dd> <dd><a target="_blank" href="https://chat.project-hop.org">Chat</a></dd> <dd><a target="_blank" href="../../community/team/">Team</a></dd> </dl> <dl> <dt>About</dt> <dd><a target="_blank" href="https://www.apache.org/licenses/" title="License">License</a></dd> </dl> <p> © 2019-2021 The Hop Team.<br> All marks mentioned may be trademarks or registered trademarks of their respective owners. </p> </div> </footer> <script src="../../_/js/site-8ed57175ed.js"></script> <script async src="../../_/js/vendor/highlight-41b831b4c4.js"></script> <script src="../../_/js/vendor/cookies-eu-banner-d4a1518c50.js"></script> <script> new CookiesEuBanner(function () { |
| (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ |
| (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), |
| m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) |
| })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); |
| |
| ga('create', 'UA-151580460-1', 'auto' ,{'anonymize_ip':true}); |
| ga('send', 'pageview'); |
| }, true, false); </script> <script src="../../_/js/vendor/docsearch-4983f0f1ec.min.js"></script> <!-- fetched from https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js --> <script> var search = docsearch({ |
| appId: 'IEDZCR8JT4', |
| apiKey: '587e7bafb978fa5e4640ab18bfbd9b99', |
| indexName: 'hop-doc', |
| inputSelector: '#search-input', |
| autocompleteOptions: { hint: false, keyboardShortcuts: ['s'] }, |
| algoliaOptions: { hitsPerPage: 10 } |
| }).autocomplete |
| search.on('autocomplete:closed', function () { search.autocomplete.setVal() }) |
| function focusSearchInput () { document.querySelector('#search-input').focus() } |
| if (document.querySelector('.home-link.is-current')) window.addEventListener('load', focusSearchInput) </script> </body> </html> |
