"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[1592],{15680:(e,t,a)=>{a.d(t,{xA:()=>g,yg:()=>c});var n=a(96540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function s(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?o(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):o(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function i(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},o=Object.keys(e);for(n=0;n<o.length;n++)a=o[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n<o.length;n++)a=o[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var l=n.createContext({}),d=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):s(s({},t),e)),a},g=function(e){var t=d(e.components);return n.createElement(l.Provider,{value:t},e.children)},u="mdxType",p={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,o=e.originalType,l=e.parentName,g=i(e,["components","mdxType","originalType","parentName"]),u=d(a),m=r,c=u["".concat(l,".").concat(m)]||u[m]||p[m]||o;return a?n.createElement(c,s(s({ref:t},g),{},{components:a})):n.createElement(c,s({ref:t},g))}));function c(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=a.length,s=new Array(o);s[0]=m;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i[u]="string"==typeof e?e:r,s[1]=i;for(var d=2;d<o;d++)s[d]=a[d];return n.createElement.apply(null,s)}return n.createElement.apply(null,a)}m.displayName="MDXCreateElement"},37736:(e,t,a)=>{a.r(t),a.d(t,{assets:()=>g,contentTitle:()=>l,default:()=>c,frontMatter:()=>i,metadata:()=>d,toc:()=>u});var n=a(58168),r=a(98587),o=(a(96540),a(15680)),s=["components"],i={id:"metadata-storage",title:"Metadata storage"},l=void 0,d={unversionedId:"design/metadata-storage",id:"design/metadata-storage",title:"Metadata storage",description:"\x3c!--",source:"@site/docs/29.0.0/design/metadata-storage.md",sourceDirName:"design",slug:"/design/metadata-storage",permalink:"/docs/29.0.0/design/metadata-storage",draft:!1,tags:[],version:"current",frontMatter:{id:"metadata-storage",title:"Metadata storage"},sidebar:"docs",previous:{title:"Deep storage",permalink:"/docs/29.0.0/design/deep-storage"},next:{title:"ZooKeeper",permalink:"/docs/29.0.0/design/zookeeper"}},g={},u=[{value:"Available metadata stores",id:"available-metadata-stores",level:2},{value:"MySQL",id:"mysql",level:3},{value:"PostgreSQL",id:"postgresql",level:3},{value:"Derby",id:"derby",level:3},{value:"Adding custom DBCP properties",id:"adding-custom-dbcp-properties",level:2},{value:"Metadata storage tables",id:"metadata-storage-tables",level:2},{value:"Segments table",id:"segments-table",level:3},{value:"Rule table",id:"rule-table",level:3},{value:"Config table",id:"config-table",level:3},{value:"Task-related tables",id:"task-related-tables",level:3},{value:"Audit table",id:"audit-table",level:3},{value:"Metadata storage access",id:"metadata-storage-access",level:2},{value:"Learn more",id:"learn-more",level:2}],p={toc:u},m="wrapper";function c(e){var t=e.components,a=(0,r.A)(e,s);return(0,o.yg)(m,(0,n.A)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"Apache Druid relies on an external dependency for metadata storage.\nDruid uses the metadata store to house various metadata about the system, but not to store the actual data.\nThe metadata store retains all metadata essential for a Druid cluster to work."),(0,o.yg)("p",null,"The metadata store includes the following:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Segments records"),(0,o.yg)("li",{parentName:"ul"},"Rule records"),(0,o.yg)("li",{parentName:"ul"},"Configuration records"),(0,o.yg)("li",{parentName:"ul"},"Task-related tables"),(0,o.yg)("li",{parentName:"ul"},"Audit records")),(0,o.yg)("p",null,"Derby is the default metadata store for Druid, however, it is not suitable for production.\n",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/development/extensions-core/mysql"},"MySQL")," and ",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/development/extensions-core/postgresql"},"PostgreSQL")," are more production suitable metadata stores.\nSee ",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/configuration/#metadata-storage"},"Metadata storage configuration")," for the default configuration settings."),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"}," We also recommend you set up a high availability environment because there is no way to restore lost metadata.")),(0,o.yg)("h2",{id:"available-metadata-stores"},"Available metadata stores"),(0,o.yg)("p",null,"Druid supports Derby, MySQL, and PostgreSQL for storing metadata."),(0,o.yg)("p",null,"To avoid issues with upgrades that require schema changes to a large metadata table, consider a metadata store version that supports instant ADD COLUMN semantics.\nSee the database-specific docs for guidance on versions."),(0,o.yg)("h3",{id:"mysql"},"MySQL"),(0,o.yg)("p",null,"See ",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/development/extensions-core/mysql"},"mysql-metadata-storage extension documentation"),"."),(0,o.yg)("h3",{id:"postgresql"},"PostgreSQL"),(0,o.yg)("p",null,"See ",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/development/extensions-core/postgresql"},"postgresql-metadata-storage"),"."),(0,o.yg)("h3",{id:"derby"},"Derby"),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"}," For production clusters, consider using MySQL or PostgreSQL instead of Derby.")),(0,o.yg)("p",null,"Configure metadata storage with Derby by setting the following properties in your Druid configuration."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-properties"},"druid.metadata.storage.type=derby\ndruid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527//opt/var/druid_state/derby;create=true\n")),(0,o.yg)("h2",{id:"adding-custom-dbcp-properties"},"Adding custom DBCP properties"),(0,o.yg)("p",null,"You can add custom properties to customize the database connection pool (DBCP) for connecting to the metadata store.\nDefine these properties with a ",(0,o.yg)("inlineCode",{parentName:"p"},"druid.metadata.storage.connector.dbcp.")," prefix.\nFor example:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-properties"},"druid.metadata.storage.connector.dbcp.maxConnLifetimeMillis=1200000\ndruid.metadata.storage.connector.dbcp.defaultQueryTimeout=30000\n")),(0,o.yg)("p",null,"Certain properties cannot be set through ",(0,o.yg)("inlineCode",{parentName:"p"},"druid.metadata.storage.connector.dbcp.")," and must be set with the prefix ",(0,o.yg)("inlineCode",{parentName:"p"},"druid.metadata.storage.connector."),":"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("inlineCode",{parentName:"li"},"username")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("inlineCode",{parentName:"li"},"password")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("inlineCode",{parentName:"li"},"connectURI")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("inlineCode",{parentName:"li"},"validationQuery")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("inlineCode",{parentName:"li"},"testOnBorrow"))),(0,o.yg)("p",null,"See ",(0,o.yg)("a",{parentName:"p",href:"https://commons.apache.org/proper/commons-dbcp/configuration"},"BasicDataSource Configuration")," for a full list of configurable properties."),(0,o.yg)("h2",{id:"metadata-storage-tables"},"Metadata storage tables"),(0,o.yg)("p",null,"This section describes the various tables in metadata storage."),(0,o.yg)("h3",{id:"segments-table"},"Segments table"),(0,o.yg)("p",null,"This is dictated by the ",(0,o.yg)("inlineCode",{parentName:"p"},"druid.metadata.storage.tables.segments")," property."),(0,o.yg)("p",null,'This table stores metadata about the segments that should be available in the system. (This set of segments is called\n"used segments" elsewhere in the documentation and throughout the project.) The table is polled by the\n',(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/design/coordinator"},"Coordinator")," to determine the set of segments that should be available for querying in the\nsystem. The table has two main functional columns, the other columns are for indexing purposes."),(0,o.yg)("p",null,"Value 1 in the ",(0,o.yg)("inlineCode",{parentName:"p"},"used"),' column means that the segment should be "used" by the cluster (i.e., it should be loaded and\navailable for requests). Value 0 means that the segment should not be loaded into the cluster. We do this as a means of\nunloading segments from the cluster without actually removing their metadata (which allows for simpler rolling back if\nthat is ever an issue). The ',(0,o.yg)("inlineCode",{parentName:"p"},"used")," column has a corresponding ",(0,o.yg)("inlineCode",{parentName:"p"},"used_status_last_updated")," column which denotes the time\nwhen the ",(0,o.yg)("inlineCode",{parentName:"p"},"used")," status of the segment was last updated. This information can be used by the Coordinator to determine if\na segment is a candidate for deletion (if automated segment killing is enabled)."),(0,o.yg)("p",null,"The ",(0,o.yg)("inlineCode",{parentName:"p"},"payload")," column stores a JSON blob that has all of the metadata for the segment.\nSome of the data in the ",(0,o.yg)("inlineCode",{parentName:"p"},"payload")," column intentionally duplicates data from other columns in the segments table.\nAs an example, the ",(0,o.yg)("inlineCode",{parentName:"p"},"payload")," column may take the following form:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-json"},'{\n "dataSource":"wikipedia",\n "interval":"2012-05-23T00:00:00.000Z/2012-05-24T00:00:00.000Z",\n "version":"2012-05-24T00:10:00.046Z",\n "loadSpec":{\n    "type":"s3_zip",\n    "bucket":"bucket_for_segment",\n    "key":"path/to/segment/on/s3"\n },\n "dimensions":"comma-delimited-list-of-dimension-names",\n "metrics":"comma-delimited-list-of-metric-names",\n "shardSpec":{"type":"none"},\n "binaryVersion":9,\n "size":size_of_segment,\n "identifier":"wikipedia_2012-05-23T00:00:00.000Z_2012-05-24T00:00:00.000Z_2012-05-23T00:10:00.046Z"\n}\n')),(0,o.yg)("h3",{id:"rule-table"},"Rule table"),(0,o.yg)("p",null,"The rule table stores the various rules about where segments should\nland. These rules are used by the ",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/design/coordinator"},"Coordinator"),"\nwhen making segment (re-)allocation decisions about the cluster."),(0,o.yg)("h3",{id:"config-table"},"Config table"),(0,o.yg)("p",null,"The config table stores runtime configuration objects. We do not have\nmany of these yet and we are not sure if we will keep this mechanism going\nforward, but it is the beginnings of a method of changing some configuration\nparameters across the cluster at runtime."),(0,o.yg)("h3",{id:"task-related-tables"},"Task-related tables"),(0,o.yg)("p",null,"Task-related tables are created and used by the ",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/design/overlord"},"Overlord")," and ",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/design/middlemanager"},"MiddleManager")," when managing tasks."),(0,o.yg)("h3",{id:"audit-table"},"Audit table"),(0,o.yg)("p",null,"The audit table stores the audit history for configuration changes\nsuch as rule changes done by ",(0,o.yg)("a",{parentName:"p",href:"/docs/29.0.0/design/coordinator"},"Coordinator")," and other\nconfig changes."),(0,o.yg)("h2",{id:"metadata-storage-access"},"Metadata storage access"),(0,o.yg)("p",null,"Only the following processes access the metadata storage:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Indexing service processes (if any)"),(0,o.yg)("li",{parentName:"ol"},"Realtime processes (if any)"),(0,o.yg)("li",{parentName:"ol"},"Coordinator processes")),(0,o.yg)("p",null,"Thus you need to give permissions (e.g., in AWS security groups) for only these machines to access the metadata storage."),(0,o.yg)("h2",{id:"learn-more"},"Learn more"),(0,o.yg)("p",null,"See the following topics for more information:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"/docs/29.0.0/configuration/#metadata-storage"},"Metadata storage configuration")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"/docs/29.0.0/operations/clean-metadata-store"},"Automated cleanup for metadata records"))))}c.isMDXComponent=!0}}]);