Google Git
Sign in
apache / atlas-website / asf-site / . / 0.6.0-incubating / Configuration.html
blob: 3f1c8decd462909da028e12233b500e20f94cbe7 [file] [log] [blame]
<!DOCTYPE html>
<!--
| Generated by Apache Maven Doxia at 2015-12-30
| Rendered using Apache Maven Fluido Skin 1.3.0
-->
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="Date-Revision-yyyymmdd" content="20151230" />
<meta http-equiv="Content-Language" content="en" />
<title>Apache Atlas &#x2013; Configuring Apache Atlas - Application Properties</title>
<link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" />
<link rel="stylesheet" href="./css/site.css" />
<link rel="stylesheet" href="./css/print.css" media="print" />
<script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script>
<script type="text/javascript">$( document ).ready( function() { $( '.carousel' ).carousel( { interval: 3500 } ) } );</script>
</head>
<body class="topBarEnabled">
<div id="topbar" class="navbar navbar-fixed-top ">
<div class="navbar-inner">
<div class="container" style="width: 68%;"><div class="nav-collapse">
<ul class="nav">
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Atlas <b class="caret"></b></a>
<ul class="dropdown-menu">
<li> <a href="index.html" title="About">About</a>
</li>
<li> <a href="https://cwiki.apache.org/confluence/display/ATLAS" title="Wiki">Wiki</a>
</li>
<li> <a href="https://cwiki.apache.org/confluence/display/ATLAS" title="News">News</a>
</li>
<li> <a href="https://git-wip-us.apache.org/repos/asf/incubator-atlas.git" title="Git">Git</a>
</li>
<li> <a href="https://issues.apache.org/jira/browse/ATLAS" title="Jira">Jira</a>
</li>
<li> <a href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy" title="Powered by">Powered by</a>
</li>
<li> <a href="http://blogs.apache.org/atlas/" title="Blog">Blog</a>
</li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Information <b class="caret"></b></a>
<ul class="dropdown-menu">
<li> <a href="project-info.html" title="Summary">Summary</a>
</li>
<li> <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a>
</li>
<li> <a href="http://webchat.freenode.net?channels=apacheatlas&uio=d4" title="IRC">IRC</a>
</li>
<li> <a href="team-list.html" title="Team">Team</a>
</li>
<li> <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a>
</li>
<li> <a href="source-repository.html" title="Source Repository">Source Repository</a>
</li>
<li> <a href="license.html" title="License">License</a>
</li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Releases <b class="caret"></b></a>
<ul class="dropdown-menu">
<li> <a href="http://www.apache.org/dyn/closer.cgi/incubator/atlas/0.5.0-incubating/" title="0.5-incubating">0.5-incubating</a>
</li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">Documentation <b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="../index.html" title="latest">latest</a></li>
<li><a href="../0.6.0-incubating/index.html" title="0.6-incubating">0.6-incubating</a></li>
<li><a href="../0.5.0-incubating/index.html" title="0.5-incubating">0.5-incubating</a></li>
</ul>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a>
<ul class="dropdown-menu">
<li> <a href="http://www.apache.org/foundation/how-it-works.html" title="How Apache Works">How Apache Works</a>
</li>
<li> <a href="http://www.apache.org/foundation/" title="Foundation">Foundation</a>
</li>
<li> <a href="http://www.apache.org/foundation/sponsorship.html" title="Sponsoring Apache">Sponsoring Apache</a>
</li>
<li> <a href="http://www.apache.org/foundation/thanks.html" title="Thanks">Thanks</a>
</li>
</ul>
</li>
</ul>
<form id="search-form" action="http://www.google.com/search" method="get" class="navbar-search pull-right" >
<input value="http://atlas.incubator.apache.org" name="sitesearch" type="hidden"/>
<input class="search-query" name="q" id="query" type="text" />
</form>
<script type="text/javascript" src="http://www.google.com/coop/cse/brand?form=search-form"></script>
<iframe src="http://www.facebook.com/plugins/like.php?href=http://atlas.incubator.apache.org/atlas-docs&send=false&layout=button_count&show-faces=false&action=like&colorscheme=dark"
scrolling="no" frameborder="0"
style="border:none; width:80px; height:20px; margin-top: 10px;" class="pull-right" ></iframe>
<script type="text/javascript" src="https://apis.google.com/js/plusone.js"></script>
<ul class="nav pull-right"><li style="margin-top: 10px;">
<div class="g-plusone" data-href="http://atlas.incubator.apache.org/atlas-docs" data-size="medium" width="60px" align="right" ></div>
</li></ul>
</div>
</div>
</div>
</div>
<div class="container">
<div id="banner">
<div class="pull-left">
<a href=".." id="bannerLeft">
<img src="images/atlas-logo.png" alt="Apache Atlas" width="200px" height="45px"/>
</a>
</div>
<div class="pull-right"> <a href="http://incubator.apache.org" id="bannerRight">
<img src="images/apache-incubator-logo.png" alt="Apache Incubator"/>
</a>
</div>
<div class="clear"><hr/></div>
</div>
<div id="breadcrumbs">
<ul class="breadcrumb">
<li class="">
<a href="http://www.apache.org" class="externalLink" title="Apache">
Apache</a>
</li>
<li class="divider ">/</li>
<li class="">
<a href="index.html" title="Atlas">
Atlas</a>
</li>
<li class="divider ">/</li>
<li class="">Configuring Apache Atlas - Application Properties</li>
<li id="publishDate" class="pull-right">Last Published: 2015-12-30</li> <li class="divider pull-right">|</li>
<li id="projectVersion" class="pull-right">Version: 0.6-incubating</li>
</ul>
</div>
<div id="bodyColumn" >
<div class="section">
<h2><a name="Configuring_Apache_Atlas_-_Application_Properties"></a>Configuring Apache Atlas - Application Properties</h2>
<p>All configuration in Atlas uses java properties style configuration. The main configuration file is application.properties which is in the <b>conf</b> dir at the deployed location. It consists of the following sections:</p></div>
<div class="section">
<h3><a name="Graph_Configs"></a>Graph Configs</h3></div>
<div class="section">
<h4><a name="Graph_persistence_engine"></a>Graph persistence engine</h4>
<p>This section sets up the graph db - titan - to use a persistence engine. Please refer to <a href="http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html">link</a> for more details. The example below uses BerkeleyDBJE.</p>
<div class="source">
<pre>
atlas.graph.storage.backend=berkeleyje
atlas.graph.storage.directory=data/berkley
</pre></div></div>
<div class="section">
<h5><a name="Graph_persistence_engine_-_Hbase"></a>Graph persistence engine - Hbase</h5>
<p>Basic configuration</p>
<div class="source">
<pre>
atlas.graph.storage.backend=hbase
#For standalone mode , specify localhost
#for distributed mode, specify zookeeper quorum here - For more information refer http://s3.thinkaurelius.com/docs/titan/current/hbase.html#_remote_server_mode_2
atlas.graph.storage.hostname=&lt;ZooKeeper Quorum&gt;
</pre></div>
<p>HBASE_CONF_DIR environment variable needs to be set to point to the Hbase client configuration directory which is added to classpath when Atlas starts up. hbase-site.xml needs to have the following properties set according to the cluster setup</p>
<div class="source">
<pre>
#Set below to /hbase-secure if the Hbase server is setup in secure mode
zookeeper.znode.parent=/hbase-unsecure
</pre></div>
<p>Advanced configuration</p>
<p># If you are planning to use any of the configs mentioned below, they need to be prefixed with &quot;atlas.graph.&quot; to take effect in ATLAS Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html#_storage_hbase">http://s3.thinkaurelius.com/docs/titan/0.5.4/titan-config-ref.html#_storage_hbase</a></p>
<p>Permissions</p>
<p>When Atlas is configured with HBase as the storage backend the graph db (titan) needs sufficient user permissions to be able to create and access an HBase table. In a secure cluster it may be necessary to grant permissions to the 'atlas' user for the 'titan' table.</p>
<p>With Ranger, a policy can be configured for 'titan'.</p>
<p>Without Ranger, HBase shell can be used to set the permissions.</p>
<div class="source">
<pre>
su hbase
kinit -k -t &lt;hbase keytab&gt; &lt;hbase principal&gt;
echo &quot;grant 'atlas', 'RWXCA', 'titan'&quot; | hbase shell
</pre></div></div>
<div class="section">
<h4><a name="Graph_Search_Index"></a>Graph Search Index</h4>
<p>This section sets up the graph db - titan - to use an search indexing system. The example configuration below sets up to use an embedded Elastic search indexing system.</p>
<div class="source">
<pre>
atlas.graph.index.search.backend=elasticsearch
atlas.graph.index.search.directory=data/es
atlas.graph.index.search.elasticsearch.client-only=false
atlas.graph.index.search.elasticsearch.local-mode=true
atlas.graph.index.search.elasticsearch.create.sleep=2000
</pre></div></div>
<div class="section">
<h5><a name="Graph_Search_Index_-_Solr"></a>Graph Search Index - Solr</h5>
<p>Please note that Solr installation in Cloud mode is a prerequisite before configuring Solr as the search indexing backend. Refer <a href="./InstallationSteps.html">InstallationSteps</a> section for Solr installation/configuration.</p>
<div class="source">
<pre>
atlas.graph.index.search.backend=solr5
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.zookeeper-url=&lt;the ZK quorum setup for solr as comma separated value&gt; eg: 10.1.6.4:2181,10.1.6.5:2181
</pre></div></div>
<div class="section">
<h4><a name="Choosing_between_Persistence_and_Indexing_Backends"></a>Choosing between Persistence and Indexing Backends</h4>
<p>Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html">http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html</a> and <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html">http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html</a> for choosing between the persistence backends. BerkeleyDB is suitable for smaller data sets in the range of upto 10 million vertices with ACID gurantees. HBase on the other hand doesnt provide ACID guarantees but is able to scale for larger graphs. HBase also provides HA inherently.</p></div>
<div class="section">
<h4><a name="Choosing_between_Persistence_Backends"></a>Choosing between Persistence Backends</h4>
<p>Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html">http://s3.thinkaurelius.com/docs/titan/0.5.4/bdb.html</a> and <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html">http://s3.thinkaurelius.com/docs/titan/0.5.4/hbase.html</a> for choosing between the persistence backends. BerkeleyDB is suitable for smaller data sets in the range of upto 10 million vertices with ACID gurantees. HBase on the other hand doesnt provide ACID guarantees but is able to scale for larger graphs. HBase also provides HA inherently.</p></div>
<div class="section">
<h4><a name="Choosing_between_Indexing_Backends"></a>Choosing between Indexing Backends</h4>
<p>Refer <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/elasticsearch.html">http://s3.thinkaurelius.com/docs/titan/0.5.4/elasticsearch.html</a> and <a class="externalLink" href="http://s3.thinkaurelius.com/docs/titan/0.5.4/solr.html">http://s3.thinkaurelius.com/docs/titan/0.5.4/solr.html</a> for chossing between <a href="./ElasticSarch.html">ElasticSarch</a> and Solr. Solr in cloud mode is the recommended setup.</p></div>
<div class="section">
<h4><a name="Switching_Persistence_Backend"></a>Switching Persistence Backend</h4>
<p>For switching the storage backend from BerkeleyDB to HBase and vice versa, refer the documentation for &quot;Graph Persistence Engine&quot; described above and restart ATLAS. The data in the indexing backend needs to be cleared else there will be discrepancies between the storage and indexing backend which could result in errors during the search. <a href="./ElasticSearch.html">ElasticSearch</a> runs by default in embedded mode and the data could easily be cleared by deleting the ATLAS_HOME/data/es directory. For Solr, the collections which were created during ATLAS Installation - vertex_index, edge_index, fulltext_index could be deleted which will cleanup the indexes</p></div>
<div class="section">
<h4><a name="Switching_Index_Backend"></a>Switching Index Backend</h4>
<p>Switching the Index backend requires clearing the persistence backend data. Otherwise there will be discrepancies between the persistence and index backends since switching the indexing backend means index data will be lost. This leads to &quot;Fulltext&quot; queries not working on the existing data For clearing the data for BerkeleyDB, delete the ATLAS_HOME/data/berkeley directory For clearing the data for HBase, in Hbase shell, run 'disable titan' and 'drop titan'</p></div>
<div class="section">
<h3><a name="Lineage_Configs"></a>Lineage Configs</h3>
<p>The higher layer services like lineage, schema, etc. are driven by the type system and this section encodes the specific types for the hive data model.</p>
<p># This models reflects the base super types for Data and Process</p>
<div class="source">
<pre>
atlas.lineage.hive.table.type.name=DataSet
atlas.lineage.hive.process.type.name=Process
atlas.lineage.hive.process.inputs.name=inputs
atlas.lineage.hive.process.outputs.name=outputs
## Schema
atlas.lineage.hive.table.schema.query=hive_table where name=?, columns
</pre></div></div>
<div class="section">
<h3><a name="Notification_Configs"></a>Notification Configs</h3>
<p>Refer <a class="externalLink" href="http://kafka.apache.org/documentation.html#configuration">http://kafka.apache.org/documentation.html#configuration</a> for Kafka configuration. All Kafka configs should be prefixed with 'atlas.kafka.'</p>
<div class="source">
<pre>
atlas.notification.embedded=true
atlas.kafka.data=${sys:atlas.home}/data/kafka
atlas.kafka.zookeeper.connect=localhost:9026
atlas.kafka.bootstrap.servers=localhost:9027
atlas.kafka.zookeeper.session.timeout.ms=400
atlas.kafka.zookeeper.sync.time.ms=20
atlas.kafka.auto.commit.interval.ms=1000
atlas.kafka.hook.group.id=atlas
</pre></div>
<p>Note that Kafka group ids are specified for a specific topic. The Kafka group id configuration for entity notifications is 'atlas.kafka.entities.group.id'</p>
<div class="source">
<pre>
atlas.kafka.entities.group.id=&lt;consumer id&gt;
</pre></div></div>
<div class="section">
<h3><a name="Client_Configs"></a>Client Configs</h3>
<div class="source">
<pre>
atlas.client.readTimeoutMSecs=60000
atlas.client.connectTimeoutMSecs=60000
atlas.rest.address=&lt;http/https&gt;://&lt;atlas-fqdn&gt;:&lt;atlas port&gt; - default http://localhost:21000
</pre></div></div>
<div class="section">
<h3><a name="Security_Properties"></a>Security Properties</h3></div>
<div class="section">
<h4><a name="SSL_config"></a>SSL config</h4>
<p>The following property is used to toggle the SSL feature.</p>
<div class="source">
<pre>
atlas.enableTLS=false
</pre></div></div>
</div>
</div>
<hr/>
<footer>
<div class="container">
<div class="row span12">Copyright &copy; 2015
<a href="http://www.apache.org">Apache Software Foundation</a>.
All Rights Reserved.
</div>
<p id="poweredBy" class="pull-right">
<a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
<img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" />
</a>
</p>
</div>
</footer>
</body>
</html>