<!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width,initial-scale=1"> <title>The Hop SDK :: Apache Hop (Incubating)</title> <link rel="canonical" href="https://hop.apache.org/dev-manual/latest/sdk/hop-sdk.html"> <meta name="generator" content="Antora 2.3.4"> <link rel="stylesheet" href="../../../_/css/site-8ea79c1e80.css"> </head> <body class="article"> <div class="cookie-banner" id="cookies-eu-banner" style="display: none;"> <div id="cookies-eu-content"> By continuing to visit this site, you accept the use of <a href="../../../cookie-info/">cookies.</a> <button id="cookies-eu-accept">Accept</button> </div> <button id="cookies-eu-reject">Reject</button> </div> <header class="header"> <nav class="navbar"> <div class="navbar-brand"> <a class="nav-logo" href="../../.."><span>Apache Hop (Incubating)</span></a> <div id="topbar-nav" class="navbar-menu"> <div class="navbar-end"> <a class="navbar-item" href="../../..../../../blog/">Blog</a> <div class="navbar-item has-dropdown is-hoverable"> <a class="navbar-link" href="#">Documentation</a> <div class="navbar-dropdown"> <a class="navbar-item" href="../../../manual/latest/getting-started.html">Getting started</a> <a class="navbar-item" href="../../../manual/latest/">User manual</a> <a class="navbar-item" href="../../../tech-manual/latest/">Technical Documentation</a> <a class="navbar-item" href="../../../dev-manual/latest/">Developer Documentation</a> <a class="navbar-item" href="../../../docs/architecture/">Architecture</a> <a class="navbar-item" href="../../../docs/roadmap/">Roadmap</a> <a class="navbar-item" href="../../../docs/qa">Q&amp;A</a> </div> </div> <div class="navbar-item has-dropdown is-hoverable"> <a class="navbar-link" href="#">Community</a> <div class="navbar-dropdown"> <a class="navbar-item" href="../../../community/contributing/">Contributing</a> <a class="navbar-item" href="../../../community/tools/">Tools</a> <a class="navbar-item" href="../../../community/team/">Team</a> </div> </div> <a class="navbar-item" href="../../..../../../download/">Download</a> <div class="navbar-item has-dropdown is-hoverable"> <a class="navbar-link" href="#">About</a> <div class="navbar-dropdown"> <a class="navbar-item" href="https://www.apache.org/licenses/LICENSE-2.0">License</a> </div> </div> </div> </div> <div class="navbar-search"> <input id="search-input" type="text" placeholder="Search docs" class="ds-input" autocomplete="off" spellcheck="false" role="combobox" aria-autocomplete="list" aria-expanded="false" aria-owns="algolia-autocomplete-listbox-0" dir="auto"> </div> <div class="navbar-tools"> <a href="https://chat.project-hop.org" title="Chat with us" target="_blank"><svg height="30" width="30" viewBox="0 0 30 30" class="brand-icon"><path d="m 23.399705,7.0517653 c -0.811583,-1.755522 -1.98643,-3.2324028 -3.542396,-4.4727237 0.04144,0.8420009 0.131904,2.6427217 0.131904,2.6427217 0,0 0.09696,0.1272 0.13632,0.182401 1.572478,2.199681 2.147037,4.6376017 1.623358,7.2857637 -1.050239,5.309843 -6.218105,8.549046 -11.455186,7.224805 C 5.6960779,18.752172 2.6254257,14.043529 3.4949926,9.3753663 4.1813918,5.6900843 6.3558852,3.2350424 9.8870488,1.9875216 l 0.1279682,-0.048 0.100416,-0.072 C 10.543401,1.2611208 10.961,0.6477608 11.406631,0 6.0547335,0.2656 0.67230798,4.5661633 0.05570071,11.107927 -0.54103457,17.438011 3.7200163,23.083134 9.8089048,24.321855 16.154114,25.612736 22.194522,21.863214 24.019864,15.86257 24.932631,12.862008 24.719032,9.9083263 23.399704,7.0542453 Z M 8.2979624,11.010647 c 0.267552,2.546882 2.4715176,4.282562 5.0209866,4.000322 2.504829,-0.2776 4.335931,-2.731601 3.861787,-5.2146427 -0.3696,-1.936001 -0.77808,-3.864402 -1.170046,-5.796084 -0.162144,-0.7991999 -0.326208,-1.5984007 -0.49824,-2.4414407 -0.06784,0.056 -0.09936,0.0848 -0.128064,0.1128 -0.858239,0.84272 -1.715997,1.6859209 -2.574236,2.5285617 -1.045056,1.02632 -2.08867,2.053921 -3.1362212,3.077522 -1.0506224,1.02664 -1.5288936,2.279601 -1.3760624,3.7327217 z"/></svg></a> <a href="https://twitter.com/projhop" title="Follow us on Twitter" target="_blank"><svg height="30" width="30" viewBox="0 0 30 30" class="brand-icon"><path d="M12 2c5.514 0 10 4.486 10 10s-4.486 10-10 10-10-4.486-10-10 4.486-10 10-10zm0-2c-6.627 0-12 5.373-12 12s5.373 12 12 12 12-5.373 12-12-5.373-12-12-12zm6.5 8.778c-.441.196-.916.328-1.414.388.509-.305.898-.787 1.083-1.362-.476.282-1.003.487-1.564.597-.448-.479-1.089-.778-1.796-.778-1.59 0-2.758 1.483-2.399 3.023-2.045-.103-3.86-1.083-5.074-2.572-.645 1.106-.334 2.554.762 3.287-.403-.013-.782-.124-1.114-.308-.027 1.14.791 2.207 1.975 2.445-.346.094-.726.116-1.112.042.313.978 1.224 1.689 2.3 1.709-1.037.812-2.34 1.175-3.647 1.021 1.09.699 2.383 1.106 3.773 1.106 4.572 0 7.154-3.861 6.998-7.324.482-.346.899-.78 1.229-1.274z"/></svg></a> <a href="https://www.linkedin.com/company/hop-project/" title="Follow us on LinkedIn" target="_blank"><svg height="30" width="30" viewBox="0 0 30 30" class="brand-icon"><path d="M19 0h-14c-2.761 0-5 2.239-5 5v14c0 2.761 2.239 5 5 5h14c2.762 0 5-2.239 5-5v-14c0-2.761-2.238-5-5-5zm-11 19h-3v-11h3v11zm-1.5-12.268c-.966 0-1.75-.79-1.75-1.764s.784-1.764 1.75-1.764 1.75.79 1.75 1.764-.783 1.764-1.75 1.764zm13.5 12.268h-3v-5.604c0-3.368-4-3.113-4 0v5.604h-3v-11h3v1.765c1.396-2.586 7-2.777 7 2.476v6.759z"/></svg></a> <a href="https://www.youtube.com/channel/UCGlcYslwe03Y2zbZ1W6DAGA" title="Subscribe to our YouTube channel" target="_blank"><svg height="30" width="30" viewBox="0 0 30 30" class="brand-icon"><path fill-rule="evenodd" d="M4.652 0h1.44l.988 3.702.916-3.702h1.454l-1.665 5.505v3.757h-1.431v-3.757l-1.702-5.505zm6.594 2.373c-1.119 0-1.861.74-1.861 1.835v3.349c0 1.204.629 1.831 1.861 1.831 1.022 0 1.826-.683 1.826-1.831v-3.349c0-1.069-.797-1.835-1.826-1.835zm.531 5.127c0 .372-.19.646-.532.646-.351 0-.554-.287-.554-.646v-3.179c0-.374.172-.651.529-.651.39 0 .557.269.557.651v3.179zm4.729-5.07v5.186c-.155.194-.5.512-.747.512-.271 0-.338-.186-.338-.46v-5.238h-1.27v5.71c0 .675.206 1.22.887 1.22.384 0 .918-.2 1.468-.853v.754h1.27v-6.831h-1.27zm2.203 13.858c-.448 0-.541.315-.541.763v.659h1.069v-.66c.001-.44-.092-.762-.528-.762zm-4.703.04c-.084.043-.167.109-.25.198v4.055c.099.106.194.182.287.229.197.1.485.107.619-.067.07-.092.105-.241.105-.449v-3.359c0-.22-.043-.386-.129-.5-.147-.193-.42-.214-.632-.107zm4.827-5.195c-2.604-.177-11.066-.177-13.666 0-2.814.192-3.146 1.892-3.167 6.367.021 4.467.35 6.175 3.167 6.367 2.6.177 11.062.177 13.666 0 2.814-.192 3.146-1.893 3.167-6.367-.021-4.467-.35-6.175-3.167-6.367zm-12.324 10.686h-1.363v-7.54h-1.41v-1.28h4.182v1.28h-1.41v7.54zm4.846 0h-1.21v-.718c-.223.265-.455.467-.696.605-.652.374-1.547.365-1.547-.955v-5.438h1.209v4.988c0 .262.063.438.322.438.236 0 .564-.303.711-.487v-4.939h1.21v6.506zm4.657-1.348c0 .805-.301 1.431-1.106 1.431-.443 0-.812-.162-1.149-.583v.5h-1.221v-8.82h1.221v2.84c.273-.333.644-.608 1.076-.608.886 0 1.18.749 1.18 1.631v3.609zm4.471-1.752h-2.314v1.228c0 .488.042.91.528.91.511 0 .541-.344.541-.91v-.452h1.245v.489c0 1.253-.538 2.013-1.813 2.013-1.155 0-1.746-.842-1.746-2.013v-2.921c0-1.129.746-1.914 1.837-1.914 1.161 0 1.721.738 1.721 1.914v1.656z"/></svg> <a href="https://github.com/project-hop/" title="Collaborate on GitHub"><svg class="brand-icon" viewBox="0 0 22 22"><path fill-rule="evenodd" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z"></path></svg></a> </a></div> <button class="navbar-burger" data-target="topbar-nav" type="button"> <span></span> <span></span> <span></span> </button> </div> </nav> </header> <div class="body"> <div class="nav-container" data-component="dev-manual" data-version="latest"> <aside class="nav"> <div class="panels"> <div class="nav-panel-menu is-active" data-panel="menu"> <nav class="nav-menu"> <h3 class="title"><a href="../index.html">Development Documentation</a></h3> <ul class="nav-list"> <li class="nav-item" data-depth="0"> <ul class="nav-list"> <li class="nav-item" data-depth="1"> <a class="nav-link" href="../getting-started.html">Getting started</a> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="../porting-kettle-plugins.html">Porting Kettle plugins</a> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="../metadata-plugins.html">Metadata plugins</a> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="../setup-dev-environment.html">Setting up your development environment</a> </li> <li class="nav-item" data-depth="1"> <a class="nav-link" href="../integration-testing.html">Integration testing</a> </li> <li class="nav-item" data-depth="1"> <button class="nav-item-toggle"></button> <a class="nav-link" href="../plugin-development.html">Plugins Development</a> <ul class="nav-list"> <li class="nav-item" data-depth="2"> <a class="nav-link" href="../start-your-own-plugin.html">Creating your own plugin</a> </li> </ul> </li> <li class="nav-item" data-depth="1"> <button class="nav-item-toggle"></button> <a class="nav-link" href="../apache-release/index.html">Apache Release Process</a> <ul class="nav-list"> <li class="nav-item" data-depth="2"> <a class="nav-link" href="../apache-release/creating-a-release.html">Creating a Release</a> </li> <li class="nav-item" data-depth="2"> <a class="nav-link" href="../apache-release/checking-a-release.html">Checking a Release</a> </li> </ul> </li> <li class="nav-item" data-depth="1"> <button class="nav-item-toggle"></button> <a class="nav-link" href="../webhop/index.html">Hop Web</a> <ul class="nav-list"> <li class="nav-item" data-depth="2"> <a class="nav-link" href="../webhop/developer-guide.html">Webhop Developer Guide</a> </li> </ul> </li> <li class="nav-item" data-depth="1"> <button class="nav-item-toggle"></button> <a class="nav-link" href="index.html">SDK</a> <ul class="nav-list"> <li class="nav-item is-current-page" data-depth="2"> <a class="nav-link" href="hop-sdk.html">The Hop SDK</a> </li> </ul> </li> </ul> </li> </ul> </nav> </div> <div class="nav-panel-explore" data-panel="explore"> <div class="context"> <span class="title">Development Documentation</span> <span class="version">latest</span> </div> <ul class="components"> <li class="component is-current"> <span class="title">Development Documentation</span> <ul class="versions"> <li class="version is-current is-latest"> <a href="../index.html">latest</a> </li> </ul> </li> <li class="component"> <span class="title">Technical Documentation</span> <ul class="versions"> <li class="version is-latest"> <a href="../../../tech-manual/latest/index.html">latest</a> </li> </ul> </li> <li class="component"> <span class="title">User manual</span> <ul class="versions"> <li class="version is-latest"> <a href="../../../manual/latest/index.html">latest</a> </li> </ul> </li> </ul> </div> </div> </aside> </div> <main> <div class="toolbar" role="navigation"> <button class="nav-toggle"></button> <a href="../../../manual/latest/index.html" class="home-link"></a> <nav class="breadcrumbs" aria-label="breadcrumbs"> <ul> <li><a href="../index.html">Development Documentation</a></li> <li><a href="index.html">SDK</a></li> <li><a href="hop-sdk.html">The Hop SDK</a></li> </ul> </nav> <div class="edit-this-page"><a href="https://github.com/apache/incubator-hop-docs/edit/asf-site/hop-dev-manual/modules/ROOT/pages/sdk/hop-sdk.adoc">Edit this Page</a></div> </div> <article class="doc"> <h1 class="page">The Hop SDK</h1> <div class="sect1"> <h2 id="_initializing"><a class="anchor" href="#_initializing"></a>Initializing</h2> <div class="sectionbody"> <div class="paragraph"> <p>First, we need to initialize the Hop API. This means we load configuration details, search for and load plugins and so on.</p> </div> <div class="paragraph"> <p>You can do this with:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-java hljs" data-lang="java">HopEnvironment.init();</code></pre> </div> </div> </div> </div> <div class="sect1"> <h2 id="_hop_metadata_providers"><a class="anchor" href="#_hop_metadata_providers"></a>Hop Metadata Providers</h2> <div class="sectionbody"> <div class="paragraph"> <p>Shared metadata in Hop is handled by a HopMetadataProvider. When using the Hop GUI you store it in a project in the <code><code>metadata/</code></code> folder. In that case you can point to such a folder with class <code><code>JsonMetadataProvider</code></code>. Note that you can serialize such a metadata collection into a single JSON string using <code><code>SerializableMetadataProvider</code></code>. This can be used to send metadata to remote servers and locations.</p> </div> <div class="paragraph"> <p>When you have a provider you can ask it to give you a serializer with which you can add and retrieve all sorts of metadata objects.</p> </div> </div> </div> <div class="sect1"> <h2 id="_variables"><a class="anchor" href="#_variables"></a>Variables</h2> <div class="sectionbody"> <div class="paragraph"> <p>If you want to work with variables in your pipelines and workflows it makes sense to create a top level IVariables object. The easiest way to do this is with:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-java hljs" data-lang="java">IVariables variables = Variables.getADefaultVariableSpace();</code></pre> </div> </div> <div class="paragraph"> <p>This method also takes into account variables which are configured in hop-config.json (if that file can be found);</p> </div> </div> </div> <div class="sect1"> <h2 id="_pipeline_metadata"><a class="anchor" href="#_pipeline_metadata"></a>Pipeline metadata</h2> <div class="sectionbody"> <div class="sect2"> <h3 id="_loading_from_a_file"><a class="anchor" href="#_loading_from_a_file"></a>Loading from a file</h3> <div class="paragraph"> <p>You can get the pipeline metadata using:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-java hljs" data-lang="java">PipelineMeta pipelineMeta = new PipelineMeta(
  "path-to-your-filename.hpl",   // The filename
  metadataProvider,             // See above
  true,                        // set internal variables
  variables                   // see above
);</code></pre> </div> </div> </div> <div class="sect2"> <h3 id="_loading_from_an_input_stream"><a class="anchor" href="#_loading_from_an_input_stream"></a>Loading from an input stream</h3> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-java hljs" data-lang="java">PipelineMeta pipelineMeta = new PipelineMeta(
  inputStream,         // The stream to load from
  metadataProvider,   // See above
  true,              // set internal variables
  variables         // see above
);</code></pre> </div> </div> </div> <div class="sect2"> <h3 id="_construct_pipeline_metadata_with_the_hop_api"><a class="anchor" href="#_construct_pipeline_metadata_with_the_hop_api"></a>Construct pipeline metadata with the Hop API</h3> <div class="paragraph"> <p>Obviously you can start with an empty pipeline and add the transforms, hops you like:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-java hljs" data-lang="java">PipelineMeta pipelineMeta = new PipelineMeta();

// Generate 1M empty rows
//
RowGeneratorMeta rowGeneratorMeta = new RowGeneratorMeta();
rowGeneratorMeta.setRowLimit("1000000");

TransformMeta rowGenerator = new TransformMeta("1M", rowGeneratorMeta);
rowGenerator.setLocation(50, 50);
pipelineMeta.addTransform(rowGenerator);

// Just a dummy placeholder for testing
//
DummyMeta dummyMeta = new DummyMeta();
TransformMeta dummy = new TransformMeta("Output", dummyMeta);
dummy.setLocation(250, 50);
pipelineMeta.addTransform(dummy);

// Add a hop between both
//
PipelineHopMeta generatorDummyHop = new PipelineHopMeta(rowGenerator, dummy);
pipelineMeta.addPipelineHop(generatorDummyHop);</code></pre> </div> </div> </div> </div> </div> <div class="sect1"> <h2 id="_pipeline_execution"><a class="anchor" href="#_pipeline_execution"></a>Pipeline execution</h2> <div class="sectionbody"> <div class="paragraph"> <p>The way a pipeline is executed depends on the run configuration you specify. To make it easy to get the engine we created a factory for you:</p> </div> <div class="listingblock"> <div class="content"> <pre class="highlightjs highlight"><code class="language-java hljs" data-lang="java">IPipelineEngine pipelineEngine = PipelineEngineFactory.createPipelineEngine(
  variables,           // see above
  "local",            // The name of the run configuration defined in the metadata
  metadataProvider,  // The metadata provider to resolve the run configuration details
  pipelineMeta      // The pipeline metadata
);

// We can now simply execute this engine...
//
pipelineEngine.execute();

// This execution runs in the background but we can wait for it to finish:
//
pipelineEngine.waitUnitlFinished();

// When it's done we can evalute the results:
//
Result result = pipelineEngine.getResult();</code></pre> </div> </div> </div> </div> </article> </main> </div> <footer> <div class="footer"> <dl> <dt>Overview</dt> <dd><a href="../../../download/">Download</a></dd> <dd><a href="../../../manual/latest/">Getting started</a></dd> </dl> <dl> <dt>Community</dt> <dd><a target="_blank" href="https://issues.apache.org/jira/projects/HOP/issues">Jira</a></dd> <dd><a target="_blank" href="https://chat.project-hop.org">Chat</a></dd> <dd><a target="_blank" href="../../../community/team/">Team</a></dd> </dl> <dl> <dt>About</dt> <dd><a target="_blank" href="https://www.apache.org/licenses/" title="License">License</a></dd> </dl> <p> &copy; 2019-2021 The Hop Team.<br> All marks mentioned may be trademarks or registered trademarks of their respective owners. </p> </div> </footer> <script src="../../../_/js/site-8ed57175ed.js"></script> <script async src="../../../_/js/vendor/highlight-41b831b4c4.js"></script> <script src="../../../_/js/vendor/cookies-eu-banner-d4a1518c50.js"></script> <script> new CookiesEuBanner(function () {
            (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
            (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
            m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
            })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
 
            ga('create', 'UA-151580460-1', 'auto' ,{'anonymize_ip':true});
            ga('send', 'pageview');
        }, true, false); </script> <script src="../../../_/js/vendor/docsearch-4983f0f1ec.min.js"></script> <!-- fetched from https://cdn.jsdelivr.net/npm/docsearch.js@2/dist/cdn/docsearch.min.js --> <script> var search = docsearch({
  appId: 'IEDZCR8JT4',
  apiKey: '587e7bafb978fa5e4640ab18bfbd9b99',
  indexName: 'hop-doc',
  inputSelector: '#search-input',
  autocompleteOptions: { hint: false, keyboardShortcuts: ['s'] },
  algoliaOptions: { hitsPerPage: 10 }
}).autocomplete
search.on('autocomplete:closed', function () { search.autocomplete.setVal() })
function focusSearchInput () { document.querySelector('#search-input').focus() }
if (document.querySelector('.home-link.is-current')) window.addEventListener('load', focusSearchInput) </script> </body> </html>