Google Git
Sign in
apache / beam-site / refs/heads/updates_release_2.8.0 / . / javadoc / 2.8.0 / org / apache / beam / sdk / io / FileBasedSink.html
blob: dd03ccfe21b4915a464fc9e78be95980c1a04e5e [file] [log] [blame]
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc (1.8.0_161-google-v7) on Thu Oct 18 16:14:35 PDT 2018 -->
<title>FileBasedSink (Apache Beam 2.8.0-SNAPSHOT)</title>
<meta name="date" content="2018-10-18">
<link rel="stylesheet" type="text/css" href="../../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../../script.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="FileBasedSink (Apache Beam 2.8.0-SNAPSHOT)";
}
}
catch(err) {
}
//-->
var methods = {"i0":9,"i1":6,"i2":10,"i3":10,"i4":10,"i5":10,"i6":10};
var tabs = {65535:["t0","All Methods"],1:["t1","Static Methods"],2:["t2","Instance Methods"],4:["t3","Abstract Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../org/apache/beam/sdk/io/DynamicFileDestinations.html" title="class in org.apache.beam.sdk.io"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.CompressionType.html" title="enum in org.apache.beam.sdk.io"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../index.html?org/apache/beam/sdk/io/FileBasedSink.html" target="_top">Frames</a></li>
<li><a href="FileBasedSink.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li><a href="#nested.class.summary">Nested</a>&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="subTitle">org.apache.beam.sdk.io</div>
<h2 title="Class FileBasedSink" class="title">Class FileBasedSink&lt;UserT,DestinationT,OutputT&gt;</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li>org.apache.beam.sdk.io.FileBasedSink&lt;UserT,DestinationT,OutputT&gt;</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<dl>
<dt><span class="paramLabel">Type Parameters:</span></dt>
<dd><code>OutputT</code> - the type of values written to the sink.</dd>
</dl>
<dl>
<dt>All Implemented Interfaces:</dt>
<dd>java.io.Serializable, <a href="../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display">HasDisplayData</a></dd>
</dl>
<hr>
<br>
<pre><a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html" title="annotation in org.apache.beam.sdk.annotations">@Experimental</a>(<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html#value--">value</a>=<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.Kind.html#FILESYSTEM">FILESYSTEM</a>)
public abstract class <span class="typeNameLabel">FileBasedSink&lt;UserT,DestinationT,OutputT&gt;</span>
extends java.lang.Object
implements java.io.Serializable, <a href="../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display">HasDisplayData</a></pre>
<div class="block">Abstract class for file-based output. An implementation of FileBasedSink writes file-based output
and defines the format of output files (how values are written, headers/footers, MIME type,
etc.).
<p>At pipeline construction time, the methods of FileBasedSink are called to validate the sink
and to create a <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WriteOperation.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.WriteOperation</code></a> that manages the process of writing to the sink.
<p>The process of writing to file-based sink is as follows:
<ol>
<li>An optional subclass-defined initialization,
<li>a parallel write of bundles to temporary files, and finally,
<li>these temporary files are renamed with final output filenames.
</ol>
<p>In order to ensure fault-tolerance, a bundle may be executed multiple times (e.g., in the
event of failure/retry or for redundancy). However, exactly one of these executions will have its
result passed to the finalize method. Each call to <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.Writer.html#open-java.lang.String-"><code>FileBasedSink.Writer.open(java.lang.String)</code></a> is passed a unique
<i>bundle id</i> when it is called by the WriteFiles transform, so even redundant or retried
bundles will have a unique way of identifying their output.
<p>The bundle id should be used to guarantee that a bundle's output is unique. This uniqueness
guarantee is important; if a bundle is to be output to a file, for example, the name of the file
will encode the unique bundle id to avoid conflicts with other writers.
<p><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a> can take a custom <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FilenamePolicy.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.FilenamePolicy</code></a> object to determine output
filenames, and this policy object can be used to write windowed or triggered PCollections into
separate files per window pane. This allows file output from unbounded PCollections, and also
works for bounded PCollecctions.
<p>Supported file systems are those registered with <a href="../../../../../org/apache/beam/sdk/io/FileSystems.html" title="class in org.apache.beam.sdk.io"><code>FileSystems</code></a>.</div>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../serialized-form.html#org.apache.beam.sdk.io.FileBasedSink">Serialized Form</a></dd>
</dl>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ======== NESTED CLASS SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="nested.class.summary">
<!-- -->
</a>
<h3>Nested Class Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Nested Class Summary table, listing nested classes, and an explanation">
<caption><span>Nested Classes</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Class and Description</th>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static class&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.CompressionType.html" title="enum in org.apache.beam.sdk.io">FileBasedSink.CompressionType</a></span></code>
<div class="block"><span class="deprecatedLabel">Deprecated.</span>&nbsp;
<div class="block"><span class="deprecationComment">use <a href="../../../../../org/apache/beam/sdk/io/Compression.html" title="enum in org.apache.beam.sdk.io"><code>Compression</code></a>.</span></div>
</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static class&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="type parameter in FileBasedSink.DynamicDestinations">UserT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="type parameter in FileBasedSink.DynamicDestinations">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="type parameter in FileBasedSink.DynamicDestinations">OutputT</a>&gt;</span></code>
<div class="block">A class that allows value-dependent writes in <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a>.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static class&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FilenamePolicy.html" title="class in org.apache.beam.sdk.io">FileBasedSink.FilenamePolicy</a></span></code>
<div class="block">A naming policy for output files.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static class&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FileResult.html" title="class in org.apache.beam.sdk.io">FileBasedSink.FileResult</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FileResult.html" title="type parameter in FileBasedSink.FileResult">DestinationT</a>&gt;</span></code>
<div class="block">Result of a single bundle write.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static class&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FileResultCoder.html" title="class in org.apache.beam.sdk.io">FileBasedSink.FileResultCoder</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FileResultCoder.html" title="type parameter in FileBasedSink.FileResultCoder">DestinationT</a>&gt;</span></code>
<div class="block">A coder for <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FileResult.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.FileResult</code></a> objects.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static interface&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.OutputFileHints.html" title="interface in org.apache.beam.sdk.io">FileBasedSink.OutputFileHints</a></span></code>
<div class="block">Provides hints about how to generate output files, such as a suggested filename suffix (e.g.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static interface&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WritableByteChannelFactory.html" title="interface in org.apache.beam.sdk.io">FileBasedSink.WritableByteChannelFactory</a></span></code>
<div class="block">Implementations create instances of <code>WritableByteChannel</code> used by <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a>
and related classes to allow <em>decorating</em>, or otherwise transforming, the raw data that
would normally be written directly to the <code>WritableByteChannel</code> passed into <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WritableByteChannelFactory.html#create-java.nio.channels.WritableByteChannel-"><code>FileBasedSink.WritableByteChannelFactory.create(WritableByteChannel)</code></a>.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static class&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WriteOperation.html" title="class in org.apache.beam.sdk.io">FileBasedSink.WriteOperation</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WriteOperation.html" title="type parameter in FileBasedSink.WriteOperation">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WriteOperation.html" title="type parameter in FileBasedSink.WriteOperation">OutputT</a>&gt;</span></code>
<div class="block">Abstract operation that manages the process of writing to <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a>.</div>
</td>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static class&nbsp;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.Writer.html" title="class in org.apache.beam.sdk.io">FileBasedSink.Writer</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.Writer.html" title="type parameter in FileBasedSink.Writer">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.Writer.html" title="type parameter in FileBasedSink.Writer">OutputT</a>&gt;</span></code>
<div class="block">Abstract writer that writes a bundle to a <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a>.</div>
</td>
</tr>
</table>
</li>
</ul>
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.summary">
<!-- -->
</a>
<h3>Constructor Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Constructor Summary table, listing constructors, and an explanation">
<caption><span>Constructors</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colOne" scope="col">Constructor and Description</th>
</tr>
<tr class="altColor">
<td class="colOne"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#FileBasedSink-org.apache.beam.sdk.options.ValueProvider-org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations-">FileBasedSink</a></span>(<a href="../../../../../org/apache/beam/sdk/options/ValueProvider.html" title="interface in org.apache.beam.sdk.options">ValueProvider</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&gt;&nbsp;tempDirectoryProvider,
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;?,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;&nbsp;dynamicDestinations)</code>
<div class="block">Construct a <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a> with the given temp directory, producing uncompressed files.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colOne"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#FileBasedSink-org.apache.beam.sdk.options.ValueProvider-org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations-org.apache.beam.sdk.io.Compression-">FileBasedSink</a></span>(<a href="../../../../../org/apache/beam/sdk/options/ValueProvider.html" title="interface in org.apache.beam.sdk.options">ValueProvider</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&gt;&nbsp;tempDirectoryProvider,
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;?,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;&nbsp;dynamicDestinations,
<a href="../../../../../org/apache/beam/sdk/io/Compression.html" title="enum in org.apache.beam.sdk.io">Compression</a>&nbsp;compression)</code>
<div class="block">Construct a <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a> with the given temp directory and output channel type.</div>
</td>
</tr>
<tr class="altColor">
<td class="colOne"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#FileBasedSink-org.apache.beam.sdk.options.ValueProvider-org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations-org.apache.beam.sdk.io.FileBasedSink.WritableByteChannelFactory-">FileBasedSink</a></span>(<a href="../../../../../org/apache/beam/sdk/options/ValueProvider.html" title="interface in org.apache.beam.sdk.options">ValueProvider</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&gt;&nbsp;tempDirectoryProvider,
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;?,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;&nbsp;dynamicDestinations,
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WritableByteChannelFactory.html" title="interface in org.apache.beam.sdk.io">FileBasedSink.WritableByteChannelFactory</a>&nbsp;writableByteChannelFactory)</code>
<div class="block">Construct a <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a> with the given temp directory and output channel type.</div>
</td>
</tr>
</table>
</li>
</ul>
<!-- ========== METHOD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t1" class="tableTab"><span><a href="javascript:show(1);">Static Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t3" class="tableTab"><span><a href="javascript:show(4);">Abstract Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code>static <a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#convertToFileResourceIfPossible-java.lang.String-">convertToFileResourceIfPossible</a></span>(java.lang.String&nbsp;outputPrefix)</code>
<div class="block">This is a helper function for turning a user-provided output filename prefix and converting it
into a <a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs"><code>ResourceId</code></a> for writing output files.</div>
</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>abstract <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WriteOperation.html" title="class in org.apache.beam.sdk.io">FileBasedSink.WriteOperation</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#createWriteOperation--">createWriteOperation</a></span>()</code>
<div class="block">Return a subclass of <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WriteOperation.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.WriteOperation</code></a> that will manage the write to the sink.</div>
</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">UserT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#getDynamicDestinations--">getDynamicDestinations</a></span>()</code>
<div class="block">Return the <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.DynamicDestinations</code></a> used.</div>
</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code><a href="../../../../../org/apache/beam/sdk/options/ValueProvider.html" title="interface in org.apache.beam.sdk.options">ValueProvider</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&gt;</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#getTempDirectoryProvider--">getTempDirectoryProvider</a></span>()</code>
<div class="block">Returns the directory inside which temprary files will be written according to the configured
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FilenamePolicy.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.FilenamePolicy</code></a>.</div>
</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code>protected <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WritableByteChannelFactory.html" title="interface in org.apache.beam.sdk.io">FileBasedSink.WritableByteChannelFactory</a></code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#getWritableByteChannelFactory--">getWritableByteChannelFactory</a></span>()</code>
<div class="block">Returns the <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WritableByteChannelFactory.html" title="interface in org.apache.beam.sdk.io"><code>FileBasedSink.WritableByteChannelFactory</code></a> used.</div>
</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a></span>(<a href="../../../../../org/apache/beam/sdk/transforms/display/DisplayData.Builder.html" title="interface in org.apache.beam.sdk.transforms.display">DisplayData.Builder</a>&nbsp;builder)</code>
<div class="block">Register display data for the given transform or component.</div>
</td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code>void</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html#validate-org.apache.beam.sdk.options.PipelineOptions-">validate</a></span>(<a href="../../../../../org/apache/beam/sdk/options/PipelineOptions.html" title="interface in org.apache.beam.sdk.options">PipelineOptions</a>&nbsp;options)</code>&nbsp;</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.detail">
<!-- -->
</a>
<h3>Constructor Detail</h3>
<a name="FileBasedSink-org.apache.beam.sdk.options.ValueProvider-org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>FileBasedSink</h4>
<pre><a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html" title="annotation in org.apache.beam.sdk.annotations">@Experimental</a>(<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html#value--">value</a>=<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.Kind.html#FILESYSTEM">FILESYSTEM</a>)
public&nbsp;FileBasedSink(<a href="../../../../../org/apache/beam/sdk/options/ValueProvider.html" title="interface in org.apache.beam.sdk.options">ValueProvider</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&gt;&nbsp;tempDirectoryProvider,
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;?,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;&nbsp;dynamicDestinations)</pre>
<div class="block">Construct a <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a> with the given temp directory, producing uncompressed files.</div>
</li>
</ul>
<a name="FileBasedSink-org.apache.beam.sdk.options.ValueProvider-org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations-org.apache.beam.sdk.io.FileBasedSink.WritableByteChannelFactory-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>FileBasedSink</h4>
<pre><a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html" title="annotation in org.apache.beam.sdk.annotations">@Experimental</a>(<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html#value--">value</a>=<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.Kind.html#FILESYSTEM">FILESYSTEM</a>)
public&nbsp;FileBasedSink(<a href="../../../../../org/apache/beam/sdk/options/ValueProvider.html" title="interface in org.apache.beam.sdk.options">ValueProvider</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&gt;&nbsp;tempDirectoryProvider,
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;?,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;&nbsp;dynamicDestinations,
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WritableByteChannelFactory.html" title="interface in org.apache.beam.sdk.io">FileBasedSink.WritableByteChannelFactory</a>&nbsp;writableByteChannelFactory)</pre>
<div class="block">Construct a <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a> with the given temp directory and output channel type.</div>
</li>
</ul>
<a name="FileBasedSink-org.apache.beam.sdk.options.ValueProvider-org.apache.beam.sdk.io.FileBasedSink.DynamicDestinations-org.apache.beam.sdk.io.Compression-">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>FileBasedSink</h4>
<pre><a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html" title="annotation in org.apache.beam.sdk.annotations">@Experimental</a>(<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html#value--">value</a>=<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.Kind.html#FILESYSTEM">FILESYSTEM</a>)
public&nbsp;FileBasedSink(<a href="../../../../../org/apache/beam/sdk/options/ValueProvider.html" title="interface in org.apache.beam.sdk.options">ValueProvider</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&gt;&nbsp;tempDirectoryProvider,
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;?,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;&nbsp;dynamicDestinations,
<a href="../../../../../org/apache/beam/sdk/io/Compression.html" title="enum in org.apache.beam.sdk.io">Compression</a>&nbsp;compression)</pre>
<div class="block">Construct a <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink</code></a> with the given temp directory and output channel type.</div>
</li>
</ul>
</li>
</ul>
<!-- ============ METHOD DETAIL ========== -->
<ul class="blockList">
<li class="blockList"><a name="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a name="convertToFileResourceIfPossible-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>convertToFileResourceIfPossible</h4>
<pre><a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html" title="annotation in org.apache.beam.sdk.annotations">@Experimental</a>(<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html#value--">value</a>=<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.Kind.html#FILESYSTEM">FILESYSTEM</a>)
public static&nbsp;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&nbsp;convertToFileResourceIfPossible(java.lang.String&nbsp;outputPrefix)</pre>
<div class="block">This is a helper function for turning a user-provided output filename prefix and converting it
into a <a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs"><code>ResourceId</code></a> for writing output files. See <a href="../../../../../org/apache/beam/sdk/io/TextIO.Write.html#to-java.lang.String-"><code>TextIO.Write.to(String)</code></a> for an
example use case.
<p>Typically, the input prefix will be something like <code>/tmp/foo/bar</code>, and the user would
like output files to be named as <code>/tmp/foo/bar-0-of-3.txt</code>. Thus, this function tries to
interpret the provided string as a file <a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs"><code>ResourceId</code></a> path.
<p>However, this may fail, for example if the user gives a prefix that is a directory. E.g.,
<code>/</code>, <code>gs://my-bucket</code>, or <code>c://</code>. In that case, interpreting the string as a
file will fail and this function will return a directory <a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs"><code>ResourceId</code></a> instead.</div>
</li>
</ul>
<a name="getDynamicDestinations--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getDynamicDestinations</h4>
<pre>public&nbsp;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io">FileBasedSink.DynamicDestinations</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">UserT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;&nbsp;getDynamicDestinations()</pre>
<div class="block">Return the <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.DynamicDestinations.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.DynamicDestinations</code></a> used.</div>
</li>
</ul>
<a name="getTempDirectoryProvider--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getTempDirectoryProvider</h4>
<pre><a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html" title="annotation in org.apache.beam.sdk.annotations">@Experimental</a>(<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.html#value--">value</a>=<a href="../../../../../org/apache/beam/sdk/annotations/Experimental.Kind.html#FILESYSTEM">FILESYSTEM</a>)
public&nbsp;<a href="../../../../../org/apache/beam/sdk/options/ValueProvider.html" title="interface in org.apache.beam.sdk.options">ValueProvider</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/fs/ResourceId.html" title="interface in org.apache.beam.sdk.io.fs">ResourceId</a>&gt;&nbsp;getTempDirectoryProvider()</pre>
<div class="block">Returns the directory inside which temprary files will be written according to the configured
<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.FilenamePolicy.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.FilenamePolicy</code></a>.</div>
</li>
</ul>
<a name="validate-org.apache.beam.sdk.options.PipelineOptions-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>validate</h4>
<pre>public&nbsp;void&nbsp;validate(<a href="../../../../../org/apache/beam/sdk/options/PipelineOptions.html" title="interface in org.apache.beam.sdk.options">PipelineOptions</a>&nbsp;options)</pre>
</li>
</ul>
<a name="createWriteOperation--">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>createWriteOperation</h4>
<pre>public abstract&nbsp;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WriteOperation.html" title="class in org.apache.beam.sdk.io">FileBasedSink.WriteOperation</a>&lt;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">DestinationT</a>,<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.html" title="type parameter in FileBasedSink">OutputT</a>&gt;&nbsp;createWriteOperation()</pre>
<div class="block">Return a subclass of <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WriteOperation.html" title="class in org.apache.beam.sdk.io"><code>FileBasedSink.WriteOperation</code></a> that will manage the write to the sink.</div>
</li>
</ul>
<a name="populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>populateDisplayData</h4>
<pre>public&nbsp;void&nbsp;populateDisplayData(<a href="../../../../../org/apache/beam/sdk/transforms/display/DisplayData.Builder.html" title="interface in org.apache.beam.sdk.transforms.display">DisplayData.Builder</a>&nbsp;builder)</pre>
<div class="block"><span class="descfrmTypeLabel">Description copied from interface:&nbsp;<code><a href="../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">HasDisplayData</a></code></span></div>
<div class="block">Register display data for the given transform or component.
<p><code>populateDisplayData(DisplayData.Builder)</code> is invoked by Pipeline runners to collect
display data via <a href="../../../../../org/apache/beam/sdk/transforms/display/DisplayData.html#from-org.apache.beam.sdk.transforms.display.HasDisplayData-"><code>DisplayData.from(HasDisplayData)</code></a>. Implementations may call <code>super.populateDisplayData(builder)</code> in order to register display data in the current namespace,
but should otherwise use <code>subcomponent.populateDisplayData(builder)</code> to use the namespace
of the subcomponent.</div>
<dl>
<dt><span class="overrideSpecifyLabel">Specified by:</span></dt>
<dd><code><a href="../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html#populateDisplayData-org.apache.beam.sdk.transforms.display.DisplayData.Builder-">populateDisplayData</a></code>&nbsp;in interface&nbsp;<code><a href="../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display">HasDisplayData</a></code></dd>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>builder</code> - The builder to populate with display data.</dd>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../org/apache/beam/sdk/transforms/display/HasDisplayData.html" title="interface in org.apache.beam.sdk.transforms.display"><code>HasDisplayData</code></a></dd>
</dl>
</li>
</ul>
<a name="getWritableByteChannelFactory--">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>getWritableByteChannelFactory</h4>
<pre>protected final&nbsp;<a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WritableByteChannelFactory.html" title="interface in org.apache.beam.sdk.io">FileBasedSink.WritableByteChannelFactory</a>&nbsp;getWritableByteChannelFactory()</pre>
<div class="block">Returns the <a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.WritableByteChannelFactory.html" title="interface in org.apache.beam.sdk.io"><code>FileBasedSink.WritableByteChannelFactory</code></a> used.</div>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<!-- ========= END OF CLASS DATA ========= -->
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../org/apache/beam/sdk/io/DynamicFileDestinations.html" title="class in org.apache.beam.sdk.io"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../../../org/apache/beam/sdk/io/FileBasedSink.CompressionType.html" title="enum in org.apache.beam.sdk.io"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../index.html?org/apache/beam/sdk/io/FileBasedSink.html" target="_top">Frames</a></li>
<li><a href="FileBasedSink.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li><a href="#nested.class.summary">Nested</a>&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</body>
</html>