blob: 28d89da951bce7a1e4019c93a8e98700efd1eb66 [file] [log] [blame]
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href='images/favicon.ico' rel='shortcut icon' type='image/x-icon'>
<!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
<title>CarbonData</title>
<style>
</style>
<!-- Bootstrap -->
<link rel="stylesheet" href="css/bootstrap.min.css">
<link href="css/style.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
<!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.scom/respond/1.4.2/respond.min.js"></script>
<![endif]-->
<script src="js/jquery.min.js"></script>
<script src="js/bootstrap.min.js"></script>
<script defer src="https://use.fontawesome.com/releases/v5.0.8/js/all.js"></script>
</head>
<body>
<header>
<nav class="navbar navbar-default navbar-custom cd-navbar-wrapper">
<div class="container">
<div class="navbar-header">
<button aria-controls="navbar" aria-expanded="false" data-target="#navbar" data-toggle="collapse"
class="navbar-toggle collapsed" type="button">
<span class="sr-only">Toggle navigation</span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
<a href="index.html" class="logo">
<img src="images/CarbonDataLogo.png" alt="CarbonData logo" title="CarbocnData logo"/>
</a>
</div>
<div class="navbar-collapse collapse cd_navcontnt" id="navbar">
<ul class="nav navbar-nav navbar-right navlist-custom">
<li><a href="index.html" class="hidden-xs"><i class="fa fa-home" aria-hidden="true"></i> </a>
</li>
<li><a href="index.html" class="hidden-lg hidden-md hidden-sm">Home</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle " data-toggle="dropdown" role="button" aria-haspopup="true"
aria-expanded="false"> Download <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/2.2.0/"
target="_blank">Apache CarbonData 2.2.0</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/2.1.1/"
target="_blank">Apache CarbonData 2.1.1</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/2.1.0/"
target="_blank">Apache CarbonData 2.1.0</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/2.0.1/"
target="_blank">Apache CarbonData 2.0.1</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/2.0.0/"
target="_blank">Apache CarbonData 2.0.0</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.1/"
target="_blank">Apache CarbonData 1.6.1</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.6.0/"
target="_blank">Apache CarbonData 1.6.0</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.4/"
target="_blank">Apache CarbonData 1.5.4</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.3/"
target="_blank">Apache CarbonData 1.5.3</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.2/"
target="_blank">Apache CarbonData 1.5.2</a></li>
<li>
<a href="https://dist.apache.org/repos/dist/release/carbondata/1.5.1/"
target="_blank">Apache CarbonData 1.5.1</a></li>
<li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/Releases"
target="_blank">Release Archive</a></li>
</ul>
</li>
<li><a href="documentation.html" class="active">Documentation</a></li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true"
aria-expanded="false">Community <span class="caret"></span></a>
<ul class="dropdown-menu">
<li>
<a href="https://github.com/apache/carbondata/blob/master/docs/how-to-contribute-to-apache-carbondata.md"
target="_blank">Contributing to CarbonData</a></li>
<li>
<a href="https://github.com/apache/carbondata/blob/master/docs/release-guide.md"
target="_blank">Release Guide</a></li>
<li>
<a href="https://cwiki.apache.org/confluence/display/CARBONDATA/PMC+and+Committers+member+list"
target="_blank">Project PMC and Committers</a></li>
<li>
<a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=66850609"
target="_blank">CarbonData Meetups</a></li>
<li><a href="security.html">Apache CarbonData Security</a></li>
<li><a href="https://issues.apache.org/jira/browse/CARBONDATA" target="_blank">Apache
Jira</a></li>
<li><a href="videogallery.html">CarbonData Videos </a></li>
</ul>
</li>
<li class="dropdown">
<a href="http://www.apache.org/" class="apache_link hidden-xs dropdown-toggle"
data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
<ul class="dropdown-menu">
<li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
<li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
<li><a href="http://www.apache.org/foundation/sponsorship.html"
target="_blank">Sponsorship</a></li>
<li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
</ul>
</li>
<li class="dropdown">
<a href="http://www.apache.org/" class="hidden-lg hidden-md hidden-sm dropdown-toggle"
data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Apache</a>
<ul class="dropdown-menu">
<li><a href="http://www.apache.org/" target="_blank">Apache Homepage</a></li>
<li><a href="http://www.apache.org/licenses/" target="_blank">License</a></li>
<li><a href="http://www.apache.org/foundation/sponsorship.html"
target="_blank">Sponsorship</a></li>
<li><a href="http://www.apache.org/foundation/thanks.html" target="_blank">Thanks</a></li>
</ul>
</li>
<li>
<a href="#" id="search-icon"><i class="fa fa-search" aria-hidden="true"></i></a>
</li>
</ul>
</div><!--/.nav-collapse -->
<div id="search-box">
<form method="get" action="http://www.google.com/search" target="_blank">
<div class="search-block">
<table border="0" cellpadding="0" width="100%">
<tr>
<td style="width:80%">
<input type="text" name="q" size=" 5" maxlength="255" value=""
class="search-input" placeholder="Search...." required/>
</td>
<td style="width:20%">
<input type="submit" value="Search"/></td>
</tr>
<tr>
<td align="left" style="font-size:75%" colspan="2">
<input type="checkbox" name="sitesearch" value="carbondata.apache.org" checked/>
<span style=" position: relative; top: -3px;"> Only search for CarbonData</span>
</td>
</tr>
</table>
</div>
</form>
</div>
</div>
</nav>
</header> <!-- end Header part -->
<div class="fixed-padding"></div> <!-- top padding with fixde header -->
<section><!-- Dashboard nav -->
<div class="container-fluid q">
<div class="col-sm-12 col-md-12 maindashboard">
<div class="verticalnavbar">
<nav class="b-sticky-nav">
<div class="nav-scroller">
<div class="nav__inner">
<a class="b-nav__intro nav__item" href="./introduction.html">introduction</a>
<a class="b-nav__quickstart nav__item" href="./quick-start-guide.html">quick start</a>
<a class="b-nav__uses nav__item" href="./usecases.html">use cases</a>
<div class="nav__item nav__item__with__subs">
<a class="b-nav__docs nav__item nav__sub__anchor" href="./language-manual.html">Language Reference</a>
<a class="nav__item nav__sub__item" href="./ddl-of-carbondata.html">DDL</a>
<a class="nav__item nav__sub__item" href="./dml-of-carbondata.html">DML</a>
<a class="nav__item nav__sub__item" href="./streaming-guide.html">Streaming</a>
<a class="nav__item nav__sub__item" href="./configuration-parameters.html">Configuration</a>
<a class="nav__item nav__sub__item" href="./index-developer-guide.html">Indexes</a>
<a class="nav__item nav__sub__item" href="./supported-data-types-in-carbondata.html">Data Types</a>
</div>
<div class="nav__item nav__item__with__subs">
<a class="b-nav__datamap nav__item nav__sub__anchor" href="./index-management.html">Index Managament</a>
<a class="nav__item nav__sub__item" href="./bloomfilter-index-guide.html">Bloom Filter</a>
<a class="nav__item nav__sub__item" href="./lucene-index-guide.html">Lucene</a>
<a class="nav__item nav__sub__item" href="./secondary-index-guide.html">Secondary Index</a>
<a class="nav__item nav__sub__item" href="../spatial-index-guide.html">Spatial Index</a>
<a class="nav__item nav__sub__item" href="../mv-guide.html">MV</a>
</div>
<div class="nav__item nav__item__with__subs">
<a class="b-nav__api nav__item nav__sub__anchor" href="./sdk-guide.html">API</a>
<a class="nav__item nav__sub__item" href="./sdk-guide.html">Java SDK</a>
<a class="nav__item nav__sub__item" href="./csdk-guide.html">C++ SDK</a>
</div>
<a class="b-nav__perf nav__item" href="./performance-tuning.html">Performance Tuning</a>
<a class="b-nav__s3 nav__item" href="./s3-guide.html">S3 Storage</a>
<a class="b-nav__indexserver nav__item" href="./index-server.html">Index Server</a>
<a class="b-nav__prestodb nav__item" href="./prestodb-guide.html">PrestoDB Integration</a>
<a class="b-nav__prestosql nav__item" href="./prestosql-guide.html">PrestoSQL Integration</a>
<a class="b-nav__flink nav__item" href="./flink-integration-guide.html">Flink Integration</a>
<a class="b-nav__scd nav__item" href="./scd-and-cdc-guide.html">SCD & CDC</a>
<a class="b-nav__faq nav__item" href="./faq.html">FAQ</a>
<a class="b-nav__contri nav__item" href="./how-to-contribute-to-apache-carbondata.html">Contribute</a>
<a class="b-nav__security nav__item" href="./security.html">Security</a>
<a class="b-nav__release nav__item" href="./release-guide.html">Release Guide</a>
</div>
</div>
<div class="navindicator">
<div class="b-nav__intro navindicator__item"></div>
<div class="b-nav__quickstart navindicator__item"></div>
<div class="b-nav__uses navindicator__item"></div>
<div class="b-nav__docs navindicator__item"></div>
<div class="b-nav__datamap navindicator__item"></div>
<div class="b-nav__api navindicator__item"></div>
<div class="b-nav__perf navindicator__item"></div>
<div class="b-nav__s3 navindicator__item"></div>
<div class="b-nav__indexserver navindicator__item"></div>
<div class="b-nav__prestodb navindicator__item"></div>
<div class="b-nav__prestosql navindicator__item"></div>
<div class="b-nav__flink navindicator__item"></div>
<div class="b-nav__scd navindicator__item"></div>
<div class="b-nav__faq navindicator__item"></div>
<div class="b-nav__contri navindicator__item"></div>
<div class="b-nav__security navindicator__item"></div>
</div>
</nav>
</div>
<div class="mdcontent">
<section>
<div style="padding:10px 15px;">
<div id="viewpage" name="viewpage">
<div class="row">
<div class="col-sm-12 col-md-12">
<div>
<h1>
<a id="alluxio-guide" class="anchor" href="#alluxio-guide" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Alluxio guide</h1>
<p>This tutorial provides a brief introduction to using Alluxio.</p>
<ul>
<li>How to use Alluxio in CarbonData?
<ul>
<li>[Running alluxio example in CarbonData project by IDEA](#Running alluxio example in CarbonData project by IDEA)</li>
<li>[CarbonData supports alluxio by spark-shell](#CarbonData supports alluxio by spark-shell)</li>
<li>[CarbonData supports alluxio by spark-submit](#CarbonData supports alluxio by spark-submit)</li>
</ul>
</li>
</ul>
<h2>
<a id="running-alluxio-example-in-carbondata-project-by-idea" class="anchor" href="#running-alluxio-example-in-carbondata-project-by-idea" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Running alluxio example in CarbonData project by IDEA</h2>
<h3>
<a id="building-carbondata" class="anchor" href="#building-carbondata" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a><a href="https://github.com/apache/carbondata/tree/master/build" target=_blank>Building CarbonData</a>
</h3>
<ul>
<li>Please refer to <a href="https://github.com/apache/carbondata/tree/master/build" target=_blank>Building CarbonData</a>.</li>
<li>Users need to install IDEA and scala plugin, and import CarbonData project.</li>
</ul>
<h3>
<a id="installing-and-starting-alluxio" class="anchor" href="#installing-and-starting-alluxio" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Installing and starting Alluxio</h3>
<ul>
<li>Please refer to <a href="https://www.alluxio.org/docs/1.8/en/Getting-Started.html#starting-alluxio" rel="nofollow">https://www.alluxio.org/docs/1.8/en/Getting-Started.html#starting-alluxio</a>
</li>
<li>Access the Alluxio web: <a href="http://localhost:19999/home" rel="nofollow">http://localhost:19999/home</a>
</li>
</ul>
<h3>
<a id="running-example" class="anchor" href="#running-example" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Running Example</h3>
<ul>
<li>Please refer to <a href="https://github.com/apache/carbondata/blob/master/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala" target=_blank>AlluxioExample</a>
</li>
</ul>
<h2>
<a id="carbondata-supports-alluxio-by-spark-shell" class="anchor" href="#carbondata-supports-alluxio-by-spark-shell" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData supports alluxio by spark-shell</h2>
<h3>
<a id="building-carbondata-1" class="anchor" href="#building-carbondata-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a><a href="https://github.com/apache/carbondata/tree/master/build" target=_blank>Building CarbonData</a>
</h3>
<ul>
<li>Please refer to <a href="https://github.com/apache/carbondata/tree/master/build" target=_blank>Building CarbonData</a>.</li>
</ul>
<h3>
<a id="preparing-spark" class="anchor" href="#preparing-spark" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Preparing Spark</h3>
<ul>
<li>Please refer to <a href="http://spark.apache.org/docs/latest/" target=_blank rel="nofollow">http://spark.apache.org/docs/latest/</a>
</li>
</ul>
<h3>
<a id="downloading-alluxio-and-uncompressing-it" class="anchor" href="#downloading-alluxio-and-uncompressing-it" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Downloading alluxio and uncompressing it</h3>
<ul>
<li>Please refer to <a href="https://www.alluxio.org/download" target=_blank rel="nofollow">https://www.alluxio.org/download</a>
</li>
</ul>
<h3>
<a id="running-spark-shell" class="anchor" href="#running-spark-shell" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Running spark-shell</h3>
<ul>
<li>Running the command in spark path</li>
</ul>
<pre lang="$command"><code>./bin/spark-shell --jars ${CARBONDATA_PATH}/assembly/target/scala-2.11/apache-carbondata-2.0.0-SNAPSHOT-bin-spark2.3.4-hadoop2.7.2.jar,${ALLUXIO_PATH}/client/alluxio-1.8.1-client.jar
</code></pre>
<ul>
<li>Testing use alluxio by CarbonSession</li>
</ul>
<pre lang="$scala"><code>import org.apache.spark.sql.CarbonSession._
import org.apache.spark.sql.SparkSession
val carbon = SparkSession.builder().master("local").appName("test").getOrCreateCarbonSession("alluxio://localhost:19998/carbondata");
carbon.sql("CREATE TABLE carbon_alluxio(id String,name String, city String,age Int) STORED as carbondata");
carbon.sql(s"LOAD DATA LOCAL INPATH '${CARBONDATA_PATH}/integration/spark/src/test/resources/sample.csv' into table carbon_alluxio");
carbon.sql("select * from carbon_alluxio").show
</code></pre>
<ul>
<li>Result</li>
</ul>
<pre lang="$scala"><code>scala&gt; carbon.sql("select * from carbon_alluxio").show
+---+------+---------+---+
| id| name| city|age|
+---+------+---------+---+
| 1| david| shenzhen| 31|
| 2| eason| shenzhen| 27|
| 3| jarry| wuhan| 35|
| 3| jarry|Bangalore| 35|
| 4| kunal| Delhi| 26|
| 4|vishal|Bangalore| 29|
+---+------+---------+---+
</code></pre>
<h2>
<a id="carbondata-supports-alluxio-by-spark-submit" class="anchor" href="#carbondata-supports-alluxio-by-spark-submit" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>CarbonData supports alluxio by spark-submit</h2>
<h3>
<a id="building-carbondata-2" class="anchor" href="#building-carbondata-2" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a><a href="https://github.com/apache/carbondata/tree/master/build" target=_blank>Building CarbonData</a>
</h3>
<ul>
<li>Please refer to <a href="https://github.com/apache/carbondata/tree/master/build" target=_blank>Building CarbonData</a>.</li>
</ul>
<h3>
<a id="preparing-spark-1" class="anchor" href="#preparing-spark-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Preparing Spark</h3>
<ul>
<li>Please refer to <a href="http://spark.apache.org/docs/latest/" target=_blank rel="nofollow">http://spark.apache.org/docs/latest/</a>
</li>
</ul>
<h3>
<a id="downloading-alluxio-and-uncompressing-it-1" class="anchor" href="#downloading-alluxio-and-uncompressing-it-1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Downloading alluxio and uncompressing it</h3>
<ul>
<li>Please refer to <a href="https://www.alluxio.org/download" target=_blank rel="nofollow">https://www.alluxio.org/download</a>
</li>
</ul>
<h3>
<a id="running-spark-submit" class="anchor" href="#running-spark-submit" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Running spark-submit</h3>
<h4>
<a id="upload-data-to-alluxio" class="anchor" href="#upload-data-to-alluxio" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Upload data to alluxio</h4>
<pre lang="$command"><code>./bin/alluxio fs copyFromLocal ${CARBONDATA_PATH}/hadoop/src/test/resources/data.csv /
</code></pre>
<h4>
<a id="command" class="anchor" href="#command" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Command</h4>
<pre lang="$command"><code>./bin/spark-submit \
--master local \
--jars ${ALLUXIO_PATH}/client/alluxio-1.8.1-client.jar,${CARBONDATA_PATH}/examples/spark/target/carbondata-examples-2.0.0-SNAPSHOT.jar \
--class org.apache.carbondata.examples.AlluxioExample \
${CARBONDATA_PATH}/assembly/target/scala-2.11/apache-carbondata-2.0.0-SNAPSHOT-bin-spark2.3.4-hadoop2.7.2.jar \
false
</code></pre>
<p><strong>NOTE</strong>: Please set runShell as false, which can avoid dependency on alluxio shell module.</p>
<h4>
<a id="result" class="anchor" href="#result" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Result</h4>
<pre lang="$command"><code>+-----------------+-------+--------------------+--------------------+---------+-----------+---------+----------+
|SegmentSequenceId| Status| Load Start Time| Load End Time|Merged To|File Format|Data Size|Index Size|
+-----------------+-------+--------------------+--------------------+---------+-----------+---------+----------+
| 1|Success|2019-01-09 15:10:...|2019-01-09 15:10:...| NA|COLUMNAR_V3| 23.92KB| 1.07KB|
| 0|Success|2019-01-09 15:10:...|2019-01-09 15:10:...| NA|COLUMNAR_V3| 23.92KB| 1.07KB|
+-----------------+-------+--------------------+--------------------+---------+-----------+---------+----------+
+-------+------+
|country|amount|
+-------+------+
| france| 202|
| china| 1698|
+-------+------+
+-----------------+---------+--------------------+--------------------+---------+-----------+---------+----------+
|SegmentSequenceId| Status| Load Start Time| Load End Time|Merged To|File Format|Data Size|Index Size|
+-----------------+---------+--------------------+--------------------+---------+-----------+---------+----------+
| 3|Compacted|2019-01-09 15:10:...|2019-01-09 15:10:...| 0.1|COLUMNAR_V3| 23.92KB| 1.03KB|
| 2|Compacted|2019-01-09 15:10:...|2019-01-09 15:10:...| 0.1|COLUMNAR_V3| 23.92KB| 1.07KB|
| 1|Compacted|2019-01-09 15:10:...|2019-01-09 15:10:...| 0.1|COLUMNAR_V3| 23.92KB| 1.07KB|
| 0.1| Success|2019-01-09 15:10:...|2019-01-09 15:10:...| NA|COLUMNAR_V3| 37.65KB| 1.08KB|
| 0|Compacted|2019-01-09 15:10:...|2019-01-09 15:10:...| 0.1|COLUMNAR_V3| 23.92KB| 1.07KB|
+-----------------+---------+--------------------+--------------------+---------+-----------+---------+----------+
</code></pre>
<h2>
<a id="reference" class="anchor" href="#reference" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Reference</h2>
<p>[1] <a href="https://www.alluxio.org/docs/1.8/en/Getting-Started.html" target=_blank rel="nofollow">https://www.alluxio.org/docs/1.8/en/Getting-Started.html</a>
[2] <a href="https://www.alluxio.org/docs/1.8/en/compute/Spark.html" target=_blank rel="nofollow">https://www.alluxio.org/docs/1.8/en/compute/Spark.html</a></p>
<script>
// Show selected style on nav item
$(function() { $('.b-nav__quickstart').addClass('selected'); });
</script></div>
</div>
</div>
</div>
<div class="doc-footer">
<a href="#top" class="scroll-top">Top</a>
</div>
</div>
</section>
</div>
</div>
</div>
</section><!-- End systemblock part -->
<script src="js/custom.js"></script>
</body>
</html>