| <!DOCTYPE html> |
| <!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> |
| <!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> |
| <head> |
| <meta charset="utf-8"> |
| <meta http-equiv="X-UA-Compatible" content="IE=edge"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| |
| <meta name="author" content="Apache Software Foundation"> |
| <link rel="shortcut icon" href="../../img/favicon.ico"> |
| <title>Config Management - Apache Gobblin</title> |
| <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'> |
| |
| <link rel="stylesheet" href="../../css/theme.css" type="text/css" /> |
| <link rel="stylesheet" href="../../css/theme_extra.css" type="text/css" /> |
| <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css"> |
| <link href="../../css/extra.css" rel="stylesheet"> |
| |
| <script> |
| // Current page data |
| var mkdocs_page_name = "Config Management"; |
| var mkdocs_page_input_path = "user-guide/Config-Management.md"; |
| var mkdocs_page_url = null; |
| </script> |
| |
| <script src="../../js/jquery-2.1.1.min.js" defer></script> |
| <script src="../../js/modernizr-2.8.3.min.js" defer></script> |
| <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script> |
| <script>hljs.initHighlightingOnLoad();</script> |
| |
| |
| </head> |
| |
| <body class="wy-body-for-nav" role="document"> |
| |
| <div class="wy-grid-for-nav"> |
| |
| |
| <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav"> |
| <div class="wy-side-nav-search"> |
| <a href="../.." class="icon icon-home"> Apache Gobblin</a> |
| <div role="search"> |
| <form id ="rtd-search-form" class="wy-form" action="../../search.html" method="get"> |
| <input type="text" name="q" placeholder="Search docs" title="Type search term here" /> |
| </form> |
| </div> |
| </div> |
| |
| <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> |
| <ul class="current"> |
| |
| |
| <li class="toctree-l1"> |
| |
| <a class="" href="/">Home</a> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <a class="" href="../../Powered-By/">Companies Powered By Gobblin</a> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <a class="" href="../../Getting-Started/">Getting Started</a> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <a class="" href="../../Gobblin-Architecture/">Architecture</a> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">User Guide</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../Working-with-Job-Configuration-Files/">Job Configuration Files</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Gobblin-Deployment/">Deployment</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Gobblin-as-a-Library/">Gobblin as a Library</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Gobblin-CLI/">Gobblin CLI</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Gobblin-Compliance/">Gobblin Compliance</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Gobblin-on-Yarn/">Gobblin on Yarn</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Compaction/">Compaction</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../State-Management-and-Watermarks/">State Management and Watermarks</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Working-with-the-ForkOperator/">Fork Operator</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Configuration-Properties-Glossary/">Configuration Glossary</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Source-schema-and-Converters/">Source schema and Converters</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Partitioned-Writers/">Partitioned Writers</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Monitoring/">Monitoring</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Gobblin-template/">Template</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Gobblin-Schedulers/">Schedulers</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Job-Execution-History-Store/">Job Execution History Store</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Building-Gobblin/">Building Gobblin</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Gobblin-genericLoad/">Generic Configuration Loading</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Hive-Registration/">Hive Registration</a> |
| </li> |
| <li class=" current"> |
| |
| <a class="current" href="./">Config Management</a> |
| <ul class="subnav"> |
| |
| <li class="toctree-l3"><a href="#table-of-contents">Table of Contents</a></li> |
| |
| |
| <li class="toctree-l3"><a href="#introduction">Introduction</a></li> |
| |
| |
| <li class="toctree-l3"><a href="#dataset-config-management-requirement">Dataset Config Management Requirement</a></li> |
| |
| <ul> |
| |
| <li><a class="toctree-l4" href="#data-model">Data Model</a></li> |
| |
| <li><a class="toctree-l4" href="#versioning">Versioning</a></li> |
| |
| <li><a class="toctree-l4" href="#client-library">Client library</a></li> |
| |
| <li><a class="toctree-l4" href="#config-store">Config Store</a></li> |
| |
| </ul> |
| |
| |
| <li class="toctree-l3"><a href="#current-dataset-config-management-implementation">Current Dataset Config Management Implementation</a></li> |
| |
| <ul> |
| |
| <li><a class="toctree-l4" href="#data-model_1">Data model</a></li> |
| |
| <li><a class="toctree-l4" href="#client-application">Client application</a></li> |
| |
| <li><a class="toctree-l4" href="#file-system-layout">File System layout</a></li> |
| |
| <li><a class="toctree-l4" href="#example-of-a-config-store">Example of a config store</a></li> |
| |
| </ul> |
| |
| |
| </ul> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Docker-Integration/">Docker Integration</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../Troubleshooting/">Troubleshooting</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../FAQs/">FAQs</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Sources</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../sources/AvroFileSource/">Avro files</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/CopySource/">File copy</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/QueryBasedSource/">Query based</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/RestApiSource/">Rest Api</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/GoogleAnalyticsSource/">Google Analytics</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/GoogleDriveSource/">Google Drive</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/GoogleWebmaster/">Google Webmaster</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/HadoopTextInputSource/">Hadoop Text Input</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/HelloWorldSource/">Hello World</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/HiveAvroToOrcSource/">Hive Avro-to-ORC</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/HivePurgerSource/">Hive compliance purging</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/SimpleJsonSource/">JSON</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/KafkaSource/">Kafka</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/MySQLSource/">MySQL</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/OracleSource/">Oracle</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/SalesforceSource/">Salesforce</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/SftpSource/">SFTP</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/SqlServerSource/">SQL Server</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/TeradataSource/">Teradata</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sources/WikipediaSource/">Wikipedia</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Sinks (Writers)</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../sinks/AvroHdfsDataWriter/">Avro HDFS</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sinks/ParquetHdfsDataWriter/">Parquet HDFS</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sinks/SimpleBytesWriter/">HDFS Byte array</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sinks/ConsoleWriter/">Console</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sinks/CouchbaseWriter/">Couchbase</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sinks/Http/">HTTP</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sinks/Gobblin-JDBC-Writer/">JDBC</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../sinks/Kafka/">Kafka</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Gobblin Adaptors</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../adaptors/Gobblin-Distcp/">Gobblin Distcp</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../adaptors/Hive-Avro-To-ORC-Converter/">Hive Avro-To-Orc Converter</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Case Studies</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../case-studies/Kafka-HDFS-Ingestion/">Kafka-HDFS Ingestion</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../case-studies/Publishing-Data-to-S3/">Publishing Data to S3</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../case-studies/Writing-ORC-Data/">Writing ORC Data</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../case-studies/Hive-Distcp/">Hive Distcp</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Gobblin Data Management</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../data-management/Gobblin-Retention/">Retention</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../data-management/DistcpNgEvents/">Distcp-NG events</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Gobblin Metrics</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../metrics/Gobblin-Metrics/">Quick Start</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../metrics/Existing-Reporters/">Existing Reporters</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../metrics/Metrics-for-Gobblin-ETL/">Metrics for Gobblin ETL</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../metrics/Gobblin-Metrics-Architecture/">Gobblin Metrics Architecture</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../metrics/Implementing-New-Reporters/">Implementing New Reporters</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../metrics/Gobblin-Metrics-Performance/">Gobblin Metrics Performance</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Developer Guide</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/Customization-for-New-Source/">Customization for New Source</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/Customization-for-Converter-and-Operator/">Customization for Converter and Operator</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/CodingStyle/">Code Style Guide</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/Gobblin-Compliance-Design/">Gobblin Compliance Design</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/IDE-setup/">IDE setup</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/Monitoring-Design/">Monitoring Design</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/Documentation-Architecture/">Documentation Architecture</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/Contributing/">Contributing</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/GobblinModules/">Gobblin Modules</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../developer-guide/HighLevelConsumer/">High Level Consumer</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Project</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../project/Feature-List/">Feature List</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="/people">Contributors and Team</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../project/Talks-and-Tech-Blogs/">Talks and Tech Blog Posts</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../project/Posts/">Posts</a> |
| </li> |
| </ul> |
| </li> |
| |
| <li class="toctree-l1"> |
| |
| <span class="caption-text">Miscellaneous</span> |
| <ul class="subnav"> |
| <li class=""> |
| |
| <a class="" href="../../miscellaneous/Camus-to-Gobblin-Migration/">Camus to Gobblin Migration</a> |
| </li> |
| <li class=""> |
| |
| <a class="" href="../../miscellaneous/Exactly-Once-Support/">Exactly Once Support</a> |
| </li> |
| </ul> |
| </li> |
| |
| </ul> |
| </div> |
| |
| </nav> |
| |
| <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> |
| |
| |
| <nav class="wy-nav-top" role="navigation" aria-label="top navigation"> |
| <i data-toggle="wy-nav-top" class="fa fa-bars"></i> |
| <a href="../..">Apache Gobblin</a> |
| </nav> |
| |
| |
| <div class="wy-nav-content"> |
| <div class="rst-content"> |
| <div role="navigation" aria-label="breadcrumbs navigation"> |
| <ul class="wy-breadcrumbs"> |
| <li><a href="../..">Docs</a> »</li> |
| |
| |
| |
| <li>User Guide »</li> |
| |
| |
| |
| <li>Config Management</li> |
| <li class="wy-breadcrumbs-aside"> |
| |
| <a href="https://github.com/apache/incubator-gobblin/edit/master/docs/user-guide/Config-Management.md" rel="nofollow"> Edit on Gobblin</a> |
| |
| </li> |
| </ul> |
| <hr/> |
| </div> |
| <div role="main"> |
| <div class="section"> |
| |
| <h1 id="table-of-contents">Table of Contents</h1> |
| <div class="toc"> |
| <ul> |
| <li><a href="#table-of-contents">Table of Contents</a></li> |
| <li><a href="#introduction">Introduction</a></li> |
| <li><a href="#dataset-config-management-requirement">Dataset Config Management Requirement</a><ul> |
| <li><a href="#data-model">Data Model</a></li> |
| <li><a href="#versioning">Versioning</a></li> |
| <li><a href="#client-library">Client library</a></li> |
| <li><a href="#config-store">Config Store</a></li> |
| </ul> |
| </li> |
| <li><a href="#current-dataset-config-management-implementation">Current Dataset Config Management Implementation</a><ul> |
| <li><a href="#data-model_1">Data model</a></li> |
| <li><a href="#client-application">Client application</a></li> |
| <li><a href="#file-system-layout">File System layout</a></li> |
| <li><a href="#example-of-a-config-store">Example of a config store</a></li> |
| </ul> |
| </li> |
| </ul> |
| </div> |
| <h1 id="introduction">Introduction</h1> |
| <p>There are multiple challenges in dataset configuration management in the context of ETL data processing as ETL infrastructure employs multi-state processing flows to ingest and publish data on HDFS. Here are some examples types of datasets and types of processing:</p> |
| <ul> |
| <li>OLTP Snapshots: ingest, publishing, replication, retention management, compliance post-processing</li> |
| <li>OLTP Increments: ingest, publishing, replication, roll-up, compaction, retention management</li> |
| <li>Streaming data: ingest, publishing, roll-up of streaming data, retention management, compliance post-processing</li> |
| <li>Opaque (derived) data: replication, retention management</li> |
| </ul> |
| <p>A typical dataset could be a database table, a Kafka topic, etc. Current the customization of the dataset processing is typically achieved through file/directory blacklists/whitelists in job/flow level configurations. This approach suffers from a number issues:</p> |
| <ul> |
| <li>Dataset unaware - control is done through low-level file/directory wildcards which can be hard to understand for more complex data layouts. </li> |
| <li>Difficulty in using/applying policies, i.e. applying the same configuration settings to large number of datasets.</li> |
| <li>Non-intuitive - the use of blacklists and whitelists can lead to properties whose effect is not always clear. </li> |
| <li>A large potential of inconsistencies across different jobs/flows.</li> |
| <li>Lack of version control</li> |
| <li>Lack of easy, aggregated view of the setup/configuration for a given dataset across all flows (including consumer access) </li> |
| </ul> |
| <p>We want to have a new way to customize the processing of each dataset like enabling/disabling certain types of processing, specific SLAs, access restrictions, retention policies, etc. without previous mentioned problems.</p> |
| <h1 id="dataset-config-management-requirement">Dataset Config Management Requirement</h1> |
| <p>Design a backend and flexible client library for storing, managing and accessing configuration that can be used to customize the process of thousands of datasets across multiple systems/flows.</p> |
| <h3 id="data-model">Data Model</h3> |
| <ul> |
| <li>(Dataset) configs are identified by a string config key</li> |
| <li>Each config key is mapped to a config object (a collection of properties)</li> |
| <li>The config object should be extensible, i.e. we should be able to add properties with arbitrary names</li> |
| <li>Hierarchical system for overrides<ul> |
| <li>Global default values</li> |
| <li>Children datasets override/inherit parent dataset configs</li> |
| <li>Ability to group properties that are specific for a group of datasets (aka tags). For example, we should be able to tag a group of Kafka datasets as "High-priority" and associate specific configuration properties with these settings.</li> |
| <li>Tags should can be applied to both datasets and other tags.</li> |
| </ul> |
| </li> |
| <li>Support for references to other properties (parameter expansion)</li> |
| <li>Late expansion of references (at time of access)</li> |
| </ul> |
| <h3 id="versioning">Versioning</h3> |
| <ul> |
| <li>Generated configs have a monotonically increasing version number</li> |
| <li>Generated configs have a date of generation</li> |
| <li>Once published configurations are immutable</li> |
| <li>Easy rollback to a previous version</li> |
| <li>Even if rolled back, a configuration is still available for processes that already use it</li> |
| <li>Audit traces about changes to configuration</li> |
| </ul> |
| <h3 id="client-library">Client library</h3> |
| <ul> |
| <li>Always loads the latest (non-rollbacked) configuration version (as of time of initialization). Once the version is fixed, the same version should be used for the remained of the processing. The consistency is needed only within a process. </li> |
| <li>Non-requirement: cross-process version consistency. The application needs to enforce consistency across processes if necessary, e.g. by copying the config to a stable location.</li> |
| <li>Ability to list the tags (policies) associated with a dataset. </li> |
| <li>Ability to list datasets associated with a tag. For example, we should be able to have a flow which can discover and process only "High-priority" datasets.</li> |
| <li>Debug info how values were derived in generated configs (e.g. from where a property value was inherited)</li> |
| </ul> |
| <h3 id="config-store">Config Store</h3> |
| <ul> |
| <li>Each config store is represented by an URI path</li> |
| <li>The URI path is significant so that configs can be associated at every level. For example, for the dataset URI hdfs_store:/data/databases/DB/Table , we should be able to associate config at every level: /data/databases/DB/Table, /data/databases/DB/, /data/databases/, etc.</li> |
| </ul> |
| <h1 id="current-dataset-config-management-implementation">Current Dataset Config Management Implementation</h1> |
| <p>At a very high-level, we extend <a href="https://github.com/typesafehub/config" rel="nofollow">typesafe config</a> with:</p> |
| <ul> |
| <li>Support for logical include URIs</li> |
| <li>Abstraction of a Config Store</li> |
| <li>Config versioning</li> |
| <li>Ability to traverse the ”import” relationships</li> |
| </ul> |
| <h3 id="data-model_1">Data model</h3> |
| <p><strong>Config key (configuration node) / config value</strong></p> |
| <p>For our use cases, we can define each configuration node per data set. All the configuration related to that dataset are specified together.</p> |
| <p>Essentially, the system provides a mapping from a config key to a config object. Each config key is represented through a URI. The config object is a map from property name to a property value. We refer to this as own config (object) and refer to it through the function own_config(K, property_name) = property_value.</p> |
| <p>A config key K can import one or more config keys I1, I2, ... . The config key K will inherit any properties from I1, I2, … that are not defined in K. The inheritance is resolved in the order of the keys I1, I2, … etc., i.e. the property will be resolved to the value in the last one that defines the property. This is similar to including configs in typesafe config. We will refer to resulting configuration as own config (object) and denote it though the function resolved_config(K, property_name) = property_value .</p> |
| <p>We also use the path in the config key URI for implicit tagging. For example, <code>/data/tracking/TOPIC</code> implicitly imports <code>/data/tracking/</code>, which implicitly imports <code>/data/</code> which implicitly imports <code>/</code>. Note that all these URI are considered as config Key so their path level implicitly indicates importation. For a given config key, all implicit imports are before the explicit imports, i.e. they have lower priority in resolution. Typical use case for this implicit importation can be a global default configuration file in root path applied to all files under it. Files in this root path can have their own setting overriding the default value inherited from root path's file. </p> |
| <p><strong>Tags</strong></p> |
| <p>For our use cases, we can define the static tags in a well known file per dataset.</p> |
| <p><strong>Dynamic tags</strong> |
| Some tags cannot be applied statically at “compile” time. For example, such are cluster-specific tags since they are on the environment where the client application runs. We will support such tags about allowing the use of limited number of variables when importing another config key. For example, such a variable can be “local_cluster.name”. Then, importing /data/tracking/${local_cluster.name} can provide cluster-specific overrides.</p> |
| <p><strong>Config Store</strong> |
| The configuration is partitioned in a number of Config Stores . Each Config Store is:</p> |
| <ul> |
| <li>mapped to a unique URI scheme;</li> |
| <li>responsible for managing the mapping of config keys (represented through URIs with the Config Store scheme) to unresolved configs;</li> |
| </ul> |
| <p align="center"> |
| <img src=../../img/configStoreDataModel.png> |
| </p> |
| |
| <h3 id="client-application">Client application</h3> |
| <p>The client application interacts using the <strong>ConfigClient</strong> API . The ConfigClient maintains a set of <strong>ConfigStoreAccessor</strong> objects which interact through the <strong>ConfigStore</strong> API with the appropriate ConfigStore implementation depending on the scheme of the ConfigStore URI . There can be a native implementation of the API like the <strong>HadoopFS ConfigStore</strong> or an adapter to an existing config/metadata store like the Hive MetaStore, etc</p> |
| <p align="center"> |
| <img src=../../img/configStoreClientApi.png> |
| </p> |
| |
| <h3 id="file-system-layout">File System layout</h3> |
| <ol> |
| <li>All configurations in one configuration store reside in it’s ROOT directory</li> |
| <li>_CONFIG_STORE file in ROOT directory (identification file for configuration store)</li> |
| <li>One or multiple version directories under ROOT</li> |
| <li>In each version, each directory represented as one configuration node<ul> |
| <li>In each directory, the main.conf file specify the configuration for that node</li> |
| <li>In each directory, the includes file specify the imports links</li> |
| </ul> |
| </li> |
| </ol> |
| <h3 id="example-of-a-config-store">Example of a config store</h3> |
| <pre><code> ROOT |
| ├── _CONFIG_STORE (contents = latest non-rolled-back version) |
| └── 1.0.53 (version directory) |
| ├── data |
| │ └── tracking |
| │ ├── TOPIC |
| │ │ ├── includes (imports links) |
| │ │ └── main.conf (configuration file) |
| │ ├── includes |
| │ └── main.conf |
| └── tags |
| ├── tracking |
| │ └── retention |
| │ └── LONG |
| │ │ ├── includes |
| │ │ └── main.conf |
| │ └── main.conf |
| └── acl |
| └── restricted |
| ├── main.conf |
| └── secdata |
| ├── includes |
| └── main.conf |
| </code></pre> |
| |
| </div> |
| </div> |
| <footer> |
| |
| <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> |
| |
| <a href="../Docker-Integration/" class="btn btn-neutral float-right" title="Docker Integration">Next <span class="icon icon-circle-arrow-right"></span></a> |
| |
| |
| <a href="../Hive-Registration/" class="btn btn-neutral" title="Hive Registration"><span class="icon icon-circle-arrow-left"></span> Previous</a> |
| |
| </div> |
| |
| |
| <hr/> |
| |
| <div role="contentinfo"> |
| <!-- Copyright etc --> |
| |
| </div> |
| |
| Built with <a href="http://www.mkdocs.org" rel="nofollow">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme" rel="nofollow">theme</a> provided by <a href="https://readthedocs.org" rel="nofollow">Read the Docs</a>. |
| </footer> |
| |
| </div> |
| </div> |
| |
| </section> |
| |
| </div> |
| |
| <div class="rst-versions" role="note" style="cursor: pointer"> |
| <span class="rst-current-version" data-toggle="rst-current-version"> |
| |
| |
| <span><a href="../Hive-Registration/" style="color: #fcfcfc;">« Previous</a></span> |
| |
| |
| <span style="margin-left: 15px"><a href="../Docker-Integration/" style="color: #fcfcfc">Next »</a></span> |
| |
| </span> |
| </div> |
| <script>var base_url = '../..';</script> |
| <script src="../../js/theme.js" defer></script> |
| <script src="../../js/extra.js" defer></script> |
| <script src="../../search/main.js" defer></script> |
| |
| </body> |
| </html> |
