blob: 18c4305a80a66d43a767724ccf22c49a66c22ec9 [file] [log] [blame]
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc -->
<title>ShardNameTemplate (Apache Beam 2.38.0-SNAPSHOT)</title>
<link rel="stylesheet" type="text/css" href="../../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../../script.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="ShardNameTemplate (Apache Beam 2.38.0-SNAPSHOT)";
}
}
catch(err) {
}
//-->
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../org/apache/beam/sdk/io/ShardingFunction.html" title="interface in org.apache.beam.sdk.io"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../../../org/apache/beam/sdk/io/Source.html" title="class in org.apache.beam.sdk.io"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../index.html?org/apache/beam/sdk/io/ShardNameTemplate.html" target="_top">Frames</a></li>
<li><a href="ShardNameTemplate.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li><a href="#field.summary">Field</a>&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#methods.inherited.from.class.java.lang.Object">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li><a href="#field.detail">Field</a>&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li>Method</li>
</ul>
</div>
<a name="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="subTitle">org.apache.beam.sdk.io</div>
<h2 title="Class ShardNameTemplate" class="title">Class ShardNameTemplate</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li>org.apache.beam.sdk.io.ShardNameTemplate</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<hr>
<br>
<pre>public class <span class="typeNameLabel">ShardNameTemplate</span>
extends java.lang.Object</pre>
<div class="block">Standard shard naming templates.
<p>Shard naming templates are strings that may contain placeholders for the shard number and
shard count. When constructing a filename for a particular shard number, the upper-case letters
'S' and 'N' are replaced with the 0-padded shard number and shard count respectively.
<p>Left-padding of the numbers enables lexicographical sorting of the resulting filenames. If the
shard number or count are too large for the space provided in the template, then the result may
no longer sort lexicographically. For example, a shard template of "S-of-N", for 200 shards, will
result in outputs named "0-of-200", ... '10-of-200', '100-of-200", etc.
<p>Shard numbers start with 0, so the last shard number is the shard count minus one. For
example, the template "-SSSSS-of-NNNNN" will be instantiated as "-00000-of-01000" for the first
shard (shard 0) of a 1000-way sharded output.
<p>A shard name template is typically provided along with a name prefix and suffix, which allows
constructing complex paths that have embedded shard information. For example, outputs in the form
"gs://bucket/path-01-of-99.txt" could be constructed by providing the individual components:
<pre><code>
pipeline.apply(
TextIO.write().to("gs://bucket/path")
.withShardNameTemplate("-SS-of-NN")
.withSuffix(".txt"))
</code></pre>
<p>In the example above, you could make parts of the output configurable by users without the
user having to specify all components of the output name.
<p>If a shard name template does not contain any repeating 'S', then the output shard count must
be 1, as otherwise the same filename would be generated for multiple shards.</div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- =========== FIELD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="field.summary">
<!-- -->
</a>
<h3>Field Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Field Summary table, listing fields, and an explanation">
<caption><span>Fields</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Field and Description</th>
</tr>
<tr class="altColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/ShardNameTemplate.html#DIRECTORY_CONTAINER">DIRECTORY_CONTAINER</a></span></code>
<div class="block">Shard is a file within a directory.</div>
</td>
</tr>
<tr class="rowColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/ShardNameTemplate.html#INDEX_OF_MAX">INDEX_OF_MAX</a></span></code>
<div class="block">Shard name containing the index and max.</div>
</td>
</tr>
</table>
</li>
</ul>
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.summary">
<!-- -->
</a>
<h3>Constructor Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Constructor Summary table, listing constructors, and an explanation">
<caption><span>Constructors</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colOne" scope="col">Constructor and Description</th>
</tr>
<tr class="altColor">
<td class="colOne"><code><span class="memberNameLink"><a href="../../../../../org/apache/beam/sdk/io/ShardNameTemplate.html#ShardNameTemplate--">ShardNameTemplate</a></span>()</code>&nbsp;</td>
</tr>
</table>
</li>
</ul>
<!-- ========== METHOD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ============ FIELD DETAIL =========== -->
<ul class="blockList">
<li class="blockList"><a name="field.detail">
<!-- -->
</a>
<h3>Field Detail</h3>
<a name="INDEX_OF_MAX">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>INDEX_OF_MAX</h4>
<pre>public static final&nbsp;java.lang.String INDEX_OF_MAX</pre>
<div class="block">Shard name containing the index and max.
<p>Eg: [prefix]-00000-of-00100[suffix] and [prefix]-00001-of-00100[suffix]</div>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.beam.sdk.io.ShardNameTemplate.INDEX_OF_MAX">Constant Field Values</a></dd>
</dl>
</li>
</ul>
<a name="DIRECTORY_CONTAINER">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>DIRECTORY_CONTAINER</h4>
<pre>public static final&nbsp;java.lang.String DIRECTORY_CONTAINER</pre>
<div class="block">Shard is a file within a directory.
<p>Eg: [prefix]/part-00000[suffix] and [prefix]/part-00001[suffix]</div>
<dl>
<dt><span class="seeLabel">See Also:</span></dt>
<dd><a href="../../../../../constant-values.html#org.apache.beam.sdk.io.ShardNameTemplate.DIRECTORY_CONTAINER">Constant Field Values</a></dd>
</dl>
</li>
</ul>
</li>
</ul>
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.detail">
<!-- -->
</a>
<h3>Constructor Detail</h3>
<a name="ShardNameTemplate--">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>ShardNameTemplate</h4>
<pre>public&nbsp;ShardNameTemplate()</pre>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<!-- ========= END OF CLASS DATA ========= -->
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../org/apache/beam/sdk/io/ShardingFunction.html" title="interface in org.apache.beam.sdk.io"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../../../org/apache/beam/sdk/io/Source.html" title="class in org.apache.beam.sdk.io"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../index.html?org/apache/beam/sdk/io/ShardNameTemplate.html" target="_top">Frames</a></li>
<li><a href="ShardNameTemplate.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li><a href="#field.summary">Field</a>&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#methods.inherited.from.class.java.lang.Object">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li><a href="#field.detail">Field</a>&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li>Method</li>
</ul>
</div>
<a name="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</body>
</html>