blob: bd517a40a1f92f72ef53a020787a77ba2649f1f8 [file] [log] [blame]
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc (1.8.0_40) on Thu Jul 20 13:15:16 PDT 2017 -->
<title>Source (gobblin-api 0.11.0 API)</title>
<meta name="date" content="2017-07-20">
<link rel="stylesheet" type="text/css" href="../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../script.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="Source (gobblin-api 0.11.0 API)";
}
}
catch(err) {
}
//-->
var methods = {"i0":6,"i1":6,"i2":6};
var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],4:["t3","Abstract Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../index-all.html">Index</a></li>
<li><a href="../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li>Prev&nbsp;Class</li>
<li><a href="../../gobblin/source/WorkUnitStreamSource.html" title="interface in gobblin.source"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../index.html?gobblin/source/Source.html" target="_top">Frames</a></li>
<li><a href="Source.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="subTitle">gobblin.source</div>
<h2 title="Interface Source" class="title">Interface Source&lt;S,D&gt;</h2>
</div>
<div class="contentContainer">
<div class="description">
<ul class="blockList">
<li class="blockList">
<dl>
<dt><span class="paramLabel">Type Parameters:</span></dt>
<dd><code>S</code> - output schema type</dd>
<dd><code>D</code> - output record type</dd>
</dl>
<dl>
<dt>All Known Subinterfaces:</dt>
<dd><a href="../../gobblin/source/WorkUnitStreamSource.html" title="interface in gobblin.source">WorkUnitStreamSource</a>&lt;S,D&gt;</dd>
</dl>
<hr>
<br>
<pre>public interface <span class="typeNameLabel">Source&lt;S,D&gt;</span></pre>
<div class="block">An interface for classes that the end users implement to work with a data source from which
schema and data records can be extracted.
<p>
An implementation of this interface should contain all the logic required to work with a
specific data source. This usually includes work determination and partitioning, and details
of the connection protocol to work with the data source.
</p></div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ========== METHOD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t3" class="tableTab"><span><a href="javascript:show(4);">Abstract Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code><a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor">Extractor</a>&lt;<a href="../../gobblin/source/Source.html" title="type parameter in Source">S</a>,<a href="../../gobblin/source/Source.html" title="type parameter in Source">D</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../gobblin/source/Source.html#getExtractor-gobblin.configuration.WorkUnitState-">getExtractor</a></span>(<a href="../../gobblin/configuration/WorkUnitState.html" title="class in gobblin.configuration">WorkUnitState</a>&nbsp;state)</code>
<div class="block">Get an <a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor"><code>Extractor</code></a> based on a given <a href="../../gobblin/configuration/WorkUnitState.html" title="class in gobblin.configuration"><code>WorkUnitState</code></a>.</div>
</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code><a href="https://docs.oracle.com/javase/7/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit">WorkUnit</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../gobblin/source/Source.html#getWorkunits-gobblin.configuration.SourceState-">getWorkunits</a></span>(<a href="../../gobblin/configuration/SourceState.html" title="class in gobblin.configuration">SourceState</a>&nbsp;state)</code>
<div class="block">Get a list of <a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit"><code>WorkUnit</code></a>s, each of which is for extracting a portion of the data.</div>
</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../gobblin/source/Source.html#shutdown-gobblin.configuration.SourceState-">shutdown</a></span>(<a href="../../gobblin/configuration/SourceState.html" title="class in gobblin.configuration">SourceState</a>&nbsp;state)</code>
<div class="block">Shutdown this <a href="../../gobblin/source/Source.html" title="interface in gobblin.source"><code>Source</code></a> instance.</div>
</td>
</tr>
</table>
</li>
</ul>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ============ METHOD DETAIL ========== -->
<ul class="blockList">
<li class="blockList"><a name="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a name="getWorkunits-gobblin.configuration.SourceState-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getWorkunits</h4>
<pre><a href="https://docs.oracle.com/javase/7/docs/api/java/util/List.html?is-external=true" title="class or interface in java.util">List</a>&lt;<a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit">WorkUnit</a>&gt;&nbsp;getWorkunits(<a href="../../gobblin/configuration/SourceState.html" title="class in gobblin.configuration">SourceState</a>&nbsp;state)</pre>
<div class="block">Get a list of <a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit"><code>WorkUnit</code></a>s, each of which is for extracting a portion of the data.
<p>
Each <a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit"><code>WorkUnit</code></a> will be used instantiate a <a href="../../gobblin/configuration/WorkUnitState.html" title="class in gobblin.configuration"><code>WorkUnitState</code></a> that gets passed to the
<a href="../../gobblin/source/Source.html#getExtractor-gobblin.configuration.WorkUnitState-"><code>getExtractor(gobblin.configuration.WorkUnitState)</code></a> method to get an <a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor"><code>Extractor</code></a> for extracting schema
and data records from the source. The <a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit"><code>WorkUnit</code></a> instance should have all the properties
needed for the <a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor"><code>Extractor</code></a> to work.
</p>
<p>
Typically the list of <a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit"><code>WorkUnit</code></a>s for the current run is determined by taking into account
the list of <a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit"><code>WorkUnit</code></a>s from the previous run so data gets extracted incrementally. The
method <code>gobblin.configuration.SourceState#getPreviousWorkUnitStates</code> can be used to get the list of <a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit"><code>WorkUnit</code></a>s
from the previous run.
</p></div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>state</code> - see <a href="../../gobblin/configuration/SourceState.html" title="class in gobblin.configuration"><code>SourceState</code></a></dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>a list of <a href="../../gobblin/source/workunit/WorkUnit.html" title="class in gobblin.source.workunit"><code>WorkUnit</code></a>s</dd>
</dl>
</li>
</ul>
<a name="getExtractor-gobblin.configuration.WorkUnitState-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getExtractor</h4>
<pre><a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor">Extractor</a>&lt;<a href="../../gobblin/source/Source.html" title="type parameter in Source">S</a>,<a href="../../gobblin/source/Source.html" title="type parameter in Source">D</a>&gt;&nbsp;getExtractor(<a href="../../gobblin/configuration/WorkUnitState.html" title="class in gobblin.configuration">WorkUnitState</a>&nbsp;state)
throws <a href="https://docs.oracle.com/javase/7/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></pre>
<div class="block">Get an <a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor"><code>Extractor</code></a> based on a given <a href="../../gobblin/configuration/WorkUnitState.html" title="class in gobblin.configuration"><code>WorkUnitState</code></a>.
<p>
The <a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor"><code>Extractor</code></a> returned can use <a href="../../gobblin/configuration/WorkUnitState.html" title="class in gobblin.configuration"><code>WorkUnitState</code></a> to store arbitrary key-value pairs
that will be persisted to the state store and loaded in the next scheduled job run.
</p></div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>state</code> - a <a href="../../gobblin/configuration/WorkUnitState.html" title="class in gobblin.configuration"><code>WorkUnitState</code></a> carrying properties needed by the returned <a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor"><code>Extractor</code></a></dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>an <a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor"><code>Extractor</code></a> used to extract schema and data records from the data source</dd>
<dt><span class="throwsLabel">Throws:</span></dt>
<dd><code><a href="https://docs.oracle.com/javase/7/docs/api/java/io/IOException.html?is-external=true" title="class or interface in java.io">IOException</a></code> - if it fails to create an <a href="../../gobblin/source/extractor/Extractor.html" title="interface in gobblin.source.extractor"><code>Extractor</code></a></dd>
</dl>
</li>
</ul>
<a name="shutdown-gobblin.configuration.SourceState-">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>shutdown</h4>
<pre>void&nbsp;shutdown(<a href="../../gobblin/configuration/SourceState.html" title="class in gobblin.configuration">SourceState</a>&nbsp;state)</pre>
<div class="block">Shutdown this <a href="../../gobblin/source/Source.html" title="interface in gobblin.source"><code>Source</code></a> instance.
<p>
This method is called once when the job completes. Properties (key-value pairs) added to the input
<a href="../../gobblin/configuration/SourceState.html" title="class in gobblin.configuration"><code>SourceState</code></a> instance will be persisted and available to the next scheduled job run through
the method <a href="../../gobblin/source/Source.html#getWorkunits-gobblin.configuration.SourceState-"><code>getWorkunits(SourceState)</code></a>. If there is no cleanup or reporting required for a
particular implementation of this interface, then it is acceptable to have a default implementation
of this method.
</p></div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>state</code> - see <a href="../../gobblin/configuration/SourceState.html" title="class in gobblin.configuration"><code>SourceState</code></a></dd>
</dl>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<!-- ========= END OF CLASS DATA ========= -->
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../index-all.html">Index</a></li>
<li><a href="../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li>Prev&nbsp;Class</li>
<li><a href="../../gobblin/source/WorkUnitStreamSource.html" title="interface in gobblin.source"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../index.html?gobblin/source/Source.html" target="_top">Frames</a></li>
<li><a href="Source.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li>Constr&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</body>
</html>