Google Git
Sign in
apache / hawq-site / de89816445f758349cdfe527ef2f3599578c778a / . / docs / pxf / javadoc / org / apache / hawq / pxf / plugins / hdfs / utilities / HdfsUtilities.html
blob: b1f7d0ddcc9a369719843672253915efde626740 [file] [log] [blame]
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<!-- NewPage -->
<html lang="en">
<head>
<!-- Generated by javadoc (1.8.0_65) on Wed Apr 20 18:03:36 PDT 2016 -->
<title>HdfsUtilities</title>
<meta name="date" content="2016-04-20">
<link rel="stylesheet" type="text/css" href="../../../../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../../../../script.js"></script>
</head>
<body>
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
parent.document.title="HdfsUtilities";
}
}
catch(err) {
}
//-->
var methods = {"i0":9,"i1":9,"i2":9,"i3":9,"i4":9,"i5":9,"i6":9,"i7":9};
var tabs = {65535:["t0","All Methods"],1:["t1","Static Methods"],8:["t4","Concrete Methods"]};
var altColor = "altColor";
var rowColor = "rowColor";
var tableTab = "tableTab";
var activeTableTab = "activeTableTab";
</script>
<noscript>
<div>JavaScript is disabled on your browser.</div>
</noscript>
<!-- ========= START OF TOP NAVBAR ======= -->
<div class="topNav"><a name="navbar.top">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.top.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/DataSchemaException.MessageFmt.html" title="enum in org.apache.hawq.pxf.plugins.hdfs.utilities"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/PxfInputFormat.html" title="class in org.apache.hawq.pxf.plugins.hdfs.utilities"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../../../index.html?org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html" target="_top">Frames</a></li>
<li><a href="HdfsUtilities.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_top">
<li><a href="../../../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_top");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.top">
<!-- -->
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<!-- ======== START OF CLASS DATA ======== -->
<div class="header">
<div class="subTitle">org.apache.hawq.pxf.plugins.hdfs.utilities</div>
<h2 title="Class HdfsUtilities" class="title">Class HdfsUtilities</h2>
</div>
<div class="contentContainer">
<ul class="inheritance">
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
<li>org.apache.hawq.pxf.plugins.hdfs.utilities.HdfsUtilities</li>
</ul>
</li>
</ul>
<div class="description">
<ul class="blockList">
<li class="blockList">
<hr>
<br>
<pre>public class <span class="typeNameLabel">HdfsUtilities</span>
extends java.lang.Object</pre>
<div class="block">HdfsUtilities class exposes helper methods for PXF classes.</div>
</li>
</ul>
</div>
<div class="summary">
<ul class="blockList">
<li class="blockList">
<!-- ======== CONSTRUCTOR SUMMARY ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.summary">
<!-- -->
</a>
<h3>Constructor Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Constructor Summary table, listing constructors, and an explanation">
<caption><span>Constructors</span><span class="tabEnd">&nbsp;</span></caption>
<tr>
<th class="colOne" scope="col">Constructor and Description</th>
</tr>
<tr class="altColor">
<td class="colOne"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#HdfsUtilities--">HdfsUtilities</a></span>()</code>&nbsp;</td>
</tr>
</table>
</li>
</ul>
<!-- ========== METHOD SUMMARY =========== -->
<ul class="blockList">
<li class="blockList"><a name="method.summary">
<!-- -->
</a>
<h3>Method Summary</h3>
<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd">&nbsp;</span></span><span id="t1" class="tableTab"><span><a href="javascript:show(1);">Static Methods</a></span><span class="tabEnd">&nbsp;</span></span><span id="t4" class="tableTab"><span><a href="javascript:show(8);">Concrete Methods</a></span><span class="tabEnd">&nbsp;</span></span></caption>
<tr>
<th class="colFirst" scope="col">Modifier and Type</th>
<th class="colLast" scope="col">Method and Description</th>
</tr>
<tr id="i0" class="altColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#absoluteDataPath-java.lang.String-">absoluteDataPath</a></span>(java.lang.String&nbsp;dataSource)</code>
<div class="block">Hdfs data sources are absolute data paths.</div>
</td>
</tr>
<tr id="i1" class="rowColor">
<td class="colFirst"><code>static org.apache.avro.Schema</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#getAvroSchema-org.apache.hadoop.conf.Configuration-java.lang.String-">getAvroSchema</a></span>(org.apache.hadoop.conf.Configuration&nbsp;conf,
java.lang.String&nbsp;dataSource)</code>
<div class="block">Accessing the Avro file through the "unsplittable" API just to get the
schema.</div>
</td>
</tr>
<tr id="i2" class="altColor">
<td class="colFirst"><code>static org.apache.hadoop.io.compress.CompressionCodec</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#getCodec-org.apache.hadoop.conf.Configuration-java.lang.String-">getCodec</a></span>(org.apache.hadoop.conf.Configuration&nbsp;conf,
java.lang.String&nbsp;name)</code>
<div class="block">Helper routine to get compression codec through reflection.</div>
</td>
</tr>
<tr id="i3" class="rowColor">
<td class="colFirst"><code>static boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#isSplittableCodec-org.apache.hadoop.fs.Path-">isSplittableCodec</a></span>(org.apache.hadoop.fs.Path&nbsp;path)</code>
<div class="block">Returns true if the needed codec is splittable.</div>
</td>
</tr>
<tr id="i4" class="altColor">
<td class="colFirst"><code>static boolean</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#isThreadSafe-java.lang.String-java.lang.String-">isThreadSafe</a></span>(java.lang.String&nbsp;dataDir,
java.lang.String&nbsp;compCodec)</code>
<div class="block">Checks if requests should be handle in a single thread or not.</div>
</td>
</tr>
<tr id="i5" class="rowColor">
<td class="colFirst"><code>static org.apache.hadoop.mapred.FileSplit</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#parseFragmentMetadata-org.apache.hawq.pxf.api.utilities.InputData-">parseFragmentMetadata</a></span>(<a href="../../../../../../../org/apache/hawq/pxf/api/utilities/InputData.html" title="class in org.apache.hawq.pxf.api.utilities">InputData</a>&nbsp;inputData)</code>
<div class="block">Parses fragment metadata and return matching <code>FileSplit</code>.</div>
</td>
</tr>
<tr id="i6" class="altColor">
<td class="colFirst"><code>static byte[]</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#prepareFragmentMetadata-org.apache.hadoop.mapred.FileSplit-">prepareFragmentMetadata</a></span>(org.apache.hadoop.mapred.FileSplit&nbsp;fsp)</code>
<div class="block">Prepares byte serialization of a file split information (start, length,
hosts) using <code>ObjectOutputStream</code>.</div>
</td>
</tr>
<tr id="i7" class="rowColor">
<td class="colFirst"><code>static java.lang.String</code></td>
<td class="colLast"><code><span class="memberNameLink"><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html#toString-java.util.List-java.lang.String-">toString</a></span>(java.util.List&lt;<a href="../../../../../../../org/apache/hawq/pxf/api/OneField.html" title="class in org.apache.hawq.pxf.api">OneField</a>&gt;&nbsp;complexRecord,
java.lang.String&nbsp;delimiter)</code>
<div class="block">Returns string serialization of list of fields.</div>
</td>
</tr>
</table>
<ul class="blockList">
<li class="blockList"><a name="methods.inherited.from.class.java.lang.Object">
<!-- -->
</a>
<h3>Methods inherited from class&nbsp;java.lang.Object</h3>
<code>clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
<div class="details">
<ul class="blockList">
<li class="blockList">
<!-- ========= CONSTRUCTOR DETAIL ======== -->
<ul class="blockList">
<li class="blockList"><a name="constructor.detail">
<!-- -->
</a>
<h3>Constructor Detail</h3>
<a name="HdfsUtilities--">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>HdfsUtilities</h4>
<pre>public&nbsp;HdfsUtilities()</pre>
</li>
</ul>
</li>
</ul>
<!-- ============ METHOD DETAIL ========== -->
<ul class="blockList">
<li class="blockList"><a name="method.detail">
<!-- -->
</a>
<h3>Method Detail</h3>
<a name="absoluteDataPath-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>absoluteDataPath</h4>
<pre>public static&nbsp;java.lang.String&nbsp;absoluteDataPath(java.lang.String&nbsp;dataSource)</pre>
<div class="block">Hdfs data sources are absolute data paths. Method ensures that dataSource
begins with '/'.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>dataSource</code> - The HDFS path to a file or directory of interest.
Retrieved from the client request.</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>an absolute data path</dd>
</dl>
</li>
</ul>
<a name="getCodec-org.apache.hadoop.conf.Configuration-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getCodec</h4>
<pre>public static&nbsp;org.apache.hadoop.io.compress.CompressionCodec&nbsp;getCodec(org.apache.hadoop.conf.Configuration&nbsp;conf,
java.lang.String&nbsp;name)</pre>
<div class="block">Helper routine to get compression codec through reflection.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>conf</code> - configuration used for reflection</dd>
<dd><code>name</code> - codec name</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>generated CompressionCodec</dd>
</dl>
</li>
</ul>
<a name="isSplittableCodec-org.apache.hadoop.fs.Path-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>isSplittableCodec</h4>
<pre>public static&nbsp;boolean&nbsp;isSplittableCodec(org.apache.hadoop.fs.Path&nbsp;path)</pre>
<div class="block">Returns true if the needed codec is splittable. If no codec is needed
returns true as well.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>path</code> - path of the file to be read</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>if the codec needed for reading the specified path is splittable.</dd>
</dl>
</li>
</ul>
<a name="isThreadSafe-java.lang.String-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>isThreadSafe</h4>
<pre>public static&nbsp;boolean&nbsp;isThreadSafe(java.lang.String&nbsp;dataDir,
java.lang.String&nbsp;compCodec)</pre>
<div class="block">Checks if requests should be handle in a single thread or not.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>dataDir</code> - hdfs path to the data source</dd>
<dd><code>compCodec</code> - the fully qualified name of the compression codec</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>if the request can be run in multi-threaded mode.</dd>
</dl>
</li>
</ul>
<a name="prepareFragmentMetadata-org.apache.hadoop.mapred.FileSplit-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>prepareFragmentMetadata</h4>
<pre>public static&nbsp;byte[]&nbsp;prepareFragmentMetadata(org.apache.hadoop.mapred.FileSplit&nbsp;fsp)
throws java.io.IOException</pre>
<div class="block">Prepares byte serialization of a file split information (start, length,
hosts) using <code>ObjectOutputStream</code>.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>fsp</code> - file split to be serialized</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>byte serialization of fsp</dd>
<dt><span class="throwsLabel">Throws:</span></dt>
<dd><code>java.io.IOException</code> - if I/O errors occur while writing to the underlying
stream</dd>
</dl>
</li>
</ul>
<a name="parseFragmentMetadata-org.apache.hawq.pxf.api.utilities.InputData-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>parseFragmentMetadata</h4>
<pre>public static&nbsp;org.apache.hadoop.mapred.FileSplit&nbsp;parseFragmentMetadata(<a href="../../../../../../../org/apache/hawq/pxf/api/utilities/InputData.html" title="class in org.apache.hawq.pxf.api.utilities">InputData</a>&nbsp;inputData)</pre>
<div class="block">Parses fragment metadata and return matching <code>FileSplit</code>.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>inputData</code> - request input data</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>FileSplit with fragment metadata</dd>
</dl>
</li>
</ul>
<a name="getAvroSchema-org.apache.hadoop.conf.Configuration-java.lang.String-">
<!-- -->
</a>
<ul class="blockList">
<li class="blockList">
<h4>getAvroSchema</h4>
<pre>public static&nbsp;org.apache.avro.Schema&nbsp;getAvroSchema(org.apache.hadoop.conf.Configuration&nbsp;conf,
java.lang.String&nbsp;dataSource)
throws java.io.IOException</pre>
<div class="block">Accessing the Avro file through the "unsplittable" API just to get the
schema. The splittable API (AvroInputFormat) which is the one we will be
using to fetch the records, does not support getting the Avro schema yet.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>conf</code> - Hadoop configuration</dd>
<dd><code>dataSource</code> - Avro file (i.e fileName.avro) path</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>the Avro schema</dd>
<dt><span class="throwsLabel">Throws:</span></dt>
<dd><code>java.io.IOException</code> - if I/O error occurred while accessing Avro schema file</dd>
</dl>
</li>
</ul>
<a name="toString-java.util.List-java.lang.String-">
<!-- -->
</a>
<ul class="blockListLast">
<li class="blockList">
<h4>toString</h4>
<pre>public static&nbsp;java.lang.String&nbsp;toString(java.util.List&lt;<a href="../../../../../../../org/apache/hawq/pxf/api/OneField.html" title="class in org.apache.hawq.pxf.api">OneField</a>&gt;&nbsp;complexRecord,
java.lang.String&nbsp;delimiter)</pre>
<div class="block">Returns string serialization of list of fields. Fields of binary type
(BYTEA) are converted to octal representation to make sure they will be
relayed properly to the DB.</div>
<dl>
<dt><span class="paramLabel">Parameters:</span></dt>
<dd><code>complexRecord</code> - list of fields to be stringified</dd>
<dd><code>delimiter</code> - delimiter between fields</dd>
<dt><span class="returnLabel">Returns:</span></dt>
<dd>string of serialized fields using delimiter</dd>
</dl>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<!-- ========= END OF CLASS DATA ========= -->
<!-- ======= START OF BOTTOM NAVBAR ====== -->
<div class="bottomNav"><a name="navbar.bottom">
<!-- -->
</a>
<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
<a name="navbar.bottom.firstrow">
<!-- -->
</a>
<ul class="navList" title="Navigation">
<li><a href="../../../../../../../overview-summary.html">Overview</a></li>
<li><a href="package-summary.html">Package</a></li>
<li class="navBarCell1Rev">Class</li>
<li><a href="package-tree.html">Tree</a></li>
<li><a href="../../../../../../../deprecated-list.html">Deprecated</a></li>
<li><a href="../../../../../../../index-all.html">Index</a></li>
<li><a href="../../../../../../../help-doc.html">Help</a></li>
</ul>
</div>
<div class="subNav">
<ul class="navList">
<li><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/DataSchemaException.MessageFmt.html" title="enum in org.apache.hawq.pxf.plugins.hdfs.utilities"><span class="typeNameLink">Prev&nbsp;Class</span></a></li>
<li><a href="../../../../../../../org/apache/hawq/pxf/plugins/hdfs/utilities/PxfInputFormat.html" title="class in org.apache.hawq.pxf.plugins.hdfs.utilities"><span class="typeNameLink">Next&nbsp;Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../../../../index.html?org/apache/hawq/pxf/plugins/hdfs/utilities/HdfsUtilities.html" target="_top">Frames</a></li>
<li><a href="HdfsUtilities.html" target="_top">No&nbsp;Frames</a></li>
</ul>
<ul class="navList" id="allclasses_navbar_bottom">
<li><a href="../../../../../../../allclasses-noframe.html">All&nbsp;Classes</a></li>
</ul>
<div>
<script type="text/javascript"><!--
allClassesLink = document.getElementById("allclasses_navbar_bottom");
if(window==top) {
allClassesLink.style.display = "block";
}
else {
allClassesLink.style.display = "none";
}
//-->
</script>
</div>
<div>
<ul class="subNavList">
<li>Summary:&nbsp;</li>
<li>Nested&nbsp;|&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.summary">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.summary">Method</a></li>
</ul>
<ul class="subNavList">
<li>Detail:&nbsp;</li>
<li>Field&nbsp;|&nbsp;</li>
<li><a href="#constructor.detail">Constr</a>&nbsp;|&nbsp;</li>
<li><a href="#method.detail">Method</a></li>
</ul>
</div>
<a name="skip.navbar.bottom">
<!-- -->
</a></div>
<!-- ======== END OF BOTTOM NAVBAR ======= -->
</body>
</html>