| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6383],{14137:(e,t,r)=>{r.d(t,{Zo:()=>c,kt:()=>h});var n=r(67294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t<arguments.length;t++){var r=null!=arguments[t]?arguments[t]:{};t%2?a(Object(r),!0).forEach((function(t){o(e,t,r[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(r)):a(Object(r)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(r,t))}))}return e}function l(e,t){if(null==e)return{};var r,n,o=function(e,t){if(null==e)return{};var r,n,o={},a=Object.keys(e);for(n=0;n<a.length;n++)r=a[n],t.indexOf(r)>=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n<a.length;n++)r=a[n],t.indexOf(r)>=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var p=n.createContext({}),s=function(e){var t=n.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=s(e.components);return n.createElement(p.Provider,{value:t},e.children)},d="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,a=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),d=s(r),m=o,h=d["".concat(p,".").concat(m)]||d[m]||u[m]||a;return r?n.createElement(h,i(i({ref:t},c),{},{components:r})):n.createElement(h,i({ref:t},c))}));function h(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=r.length,i=new Array(a);i[0]=m;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l[d]="string"==typeof e?e:o,i[1]=l;for(var s=2;s<a;s++)i[s]=r[s];return n.createElement.apply(null,i)}return n.createElement.apply(null,r)}m.displayName="MDXCreateElement"},447:(e,t,r)=>{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>h,frontMatter:()=>l,metadata:()=>s,toc:()=>d});var n=r(87462),o=r(63366),a=(r(67294),r(14137)),i=["components"],l={id:"zookeeper",title:"ZooKeeper"},p=void 0,s={unversionedId:"design/zookeeper",id:"design/zookeeper",title:"ZooKeeper",description:"\x3c!--",source:"@site/docs/31.0.0/design/zookeeper.md",sourceDirName:"design",slug:"/design/zookeeper",permalink:"/docs/31.0.0/design/zookeeper",draft:!1,tags:[],version:"current",frontMatter:{id:"zookeeper",title:"ZooKeeper"},sidebar:"docs",previous:{title:"Metadata storage",permalink:"/docs/31.0.0/design/metadata-storage"},next:{title:"Overview",permalink:"/docs/31.0.0/ingestion/"}},c={},d=[{value:"Minimum ZooKeeper versions",id:"minimum-zookeeper-versions",level:2},{value:"ZooKeeper Operations",id:"zookeeper-operations",level:2},{value:"Coordinator Leader Election",id:"coordinator-leader-election",level:2},{value:"Segment "publishing" protocol from Historical and Realtime",id:"segment-publishing-protocol-from-historical-and-realtime",level:2}],u={toc:d},m="wrapper";function h(e){var t=e.components,r=(0,o.Z)(e,i);return(0,a.kt)(m,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("p",null,"Apache Druid uses ",(0,a.kt)("a",{parentName:"p",href:"http://zookeeper.apache.org/"},"Apache ZooKeeper")," (ZK) for management of current cluster state."),(0,a.kt)("h2",{id:"minimum-zookeeper-versions"},"Minimum ZooKeeper versions"),(0,a.kt)("p",null,"Apache Druid supports ZooKeeper versions 3.5.x and above."),(0,a.kt)("admonition",{type:"info"},(0,a.kt)("p",{parentName:"admonition"}," Note: Starting with Apache Druid 0.22.0, support for ZooKeeper 3.4.x has been removed\nStarting with Apache Druid 31.0.0, support for Zookeeper-based segment loading has been removed.")),(0,a.kt)("h2",{id:"zookeeper-operations"},"ZooKeeper Operations"),(0,a.kt)("p",null,"The operations that happen over ZK are"),(0,a.kt)("ol",null,(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"/docs/31.0.0/design/coordinator"},"Coordinator")," leader election"),(0,a.kt)("li",{parentName:"ol"},'Segment "publishing" protocol from ',(0,a.kt)("a",{parentName:"li",href:"/docs/31.0.0/design/historical"},"Historical")),(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"/docs/31.0.0/design/overlord"},"Overlord")," leader election"),(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"/docs/31.0.0/design/overlord"},"Overlord")," and ",(0,a.kt)("a",{parentName:"li",href:"/docs/31.0.0/design/middlemanager"},"Middle Manager")," task management")),(0,a.kt)("h2",{id:"coordinator-leader-election"},"Coordinator Leader Election"),(0,a.kt)("p",null,"We use the Curator ",(0,a.kt)("a",{parentName:"p",href:"https://curator.apache.org/curator-recipes/leader-latch.html"},"LeaderLatch")," recipe to perform leader election at path"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},"${druid.zk.paths.coordinatorPath}/_COORDINATOR\n")),(0,a.kt)("h2",{id:"segment-publishing-protocol-from-historical-and-realtime"},'Segment "publishing" protocol from Historical and Realtime'),(0,a.kt)("p",null,"The ",(0,a.kt)("inlineCode",{parentName:"p"},"announcementsPath")," and ",(0,a.kt)("inlineCode",{parentName:"p"},"liveSegmentsPath")," are used for this."),(0,a.kt)("p",null,"All ",(0,a.kt)("a",{parentName:"p",href:"/docs/31.0.0/design/historical"},"Historical")," processes publish themselves on the ",(0,a.kt)("inlineCode",{parentName:"p"},"announcementsPath"),", specifically, they will create an ephemeral znode at"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},"${druid.zk.paths.announcementsPath}/${druid.host}\n")),(0,a.kt)("p",null,"Which signifies that they exist. They will also subsequently create a permanent znode at"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},"${druid.zk.paths.liveSegmentsPath}/${druid.host}\n")),(0,a.kt)("p",null,"And as they load up segments, they will attach ephemeral znodes that look like"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre"},"${druid.zk.paths.liveSegmentsPath}/${druid.host}/_segment_identifier_\n")),(0,a.kt)("p",null,"Processes like the ",(0,a.kt)("a",{parentName:"p",href:"/docs/31.0.0/design/coordinator"},"Coordinator")," and ",(0,a.kt)("a",{parentName:"p",href:"/docs/31.0.0/design/broker"},"Broker")," can then watch these paths to see which processes are currently serving which segments."))}h.isMDXComponent=!0}}]); |