| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5506],{28453:(e,r,n)=>{n.d(r,{R:()=>t,x:()=>d});var o=n(96540);const i={},s=o.createContext(i);function t(e){const r=o.useContext(s);return o.useMemo((function(){return"function"==typeof e?e(r):{...r,...e}}),[r,e])}function d(e){let r;return r=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:t(e.components),o.createElement(s.Provider,{value:r},e.children)}},61871:(e,r,n)=>{n.r(r),n.d(r,{assets:()=>a,contentTitle:()=>d,default:()=>h,frontMatter:()=>t,metadata:()=>o,toc:()=>c});const o=JSON.parse('{"id":"design/zookeeper","title":"ZooKeeper","description":"\x3c!--","source":"@site/docs/33.0.0/design/zookeeper.md","sourceDirName":"design","slug":"/design/zookeeper","permalink":"/docs/33.0.0/design/zookeeper","draft":false,"unlisted":false,"tags":[],"version":"current","frontMatter":{"id":"zookeeper","title":"ZooKeeper"},"sidebar":"docs","previous":{"title":"Metadata storage","permalink":"/docs/33.0.0/design/metadata-storage"},"next":{"title":"Overview","permalink":"/docs/33.0.0/ingestion/"}}');var i=n(74848),s=n(28453);const t={id:"zookeeper",title:"ZooKeeper"},d=void 0,a={},c=[{value:"Minimum ZooKeeper versions",id:"minimum-zookeeper-versions",level:2},{value:"ZooKeeper Operations",id:"zookeeper-operations",level:2},{value:"Coordinator Leader Election",id:"coordinator-leader-election",level:2},{value:"Segment "publishing" protocol from Historical and Realtime",id:"segment-publishing-protocol-from-historical-and-realtime",level:2}];function l(e){const r={a:"a",admonition:"admonition",code:"code",h2:"h2",li:"li",ol:"ol",p:"p",pre:"pre",...(0,s.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsxs)(r.p,{children:["Apache Druid uses ",(0,i.jsx)(r.a,{href:"http://zookeeper.apache.org/",children:"Apache ZooKeeper"})," (ZK) for management of current cluster state."]}),"\n",(0,i.jsx)(r.h2,{id:"minimum-zookeeper-versions",children:"Minimum ZooKeeper versions"}),"\n",(0,i.jsx)(r.p,{children:"Apache Druid supports ZooKeeper versions 3.5.x and above."}),"\n",(0,i.jsx)(r.admonition,{type:"info",children:(0,i.jsx)(r.p,{children:"Note: Starting with Apache Druid 0.22.0, support for ZooKeeper 3.4.x has been removed\nStarting with Apache Druid 31.0.0, support for Zookeeper-based segment loading has been removed."})}),"\n",(0,i.jsx)(r.h2,{id:"zookeeper-operations",children:"ZooKeeper Operations"}),"\n",(0,i.jsx)(r.p,{children:"The operations that happen over ZK are"}),"\n",(0,i.jsxs)(r.ol,{children:["\n",(0,i.jsxs)(r.li,{children:[(0,i.jsx)(r.a,{href:"/docs/33.0.0/design/coordinator",children:"Coordinator"})," leader election"]}),"\n",(0,i.jsxs)(r.li,{children:['Segment "publishing" protocol from ',(0,i.jsx)(r.a,{href:"/docs/33.0.0/design/historical",children:"Historical"})]}),"\n",(0,i.jsxs)(r.li,{children:[(0,i.jsx)(r.a,{href:"/docs/33.0.0/design/overlord",children:"Overlord"})," leader election"]}),"\n",(0,i.jsxs)(r.li,{children:[(0,i.jsx)(r.a,{href:"/docs/33.0.0/design/overlord",children:"Overlord"})," and ",(0,i.jsx)(r.a,{href:"/docs/33.0.0/design/middlemanager",children:"Middle Manager"})," task management"]}),"\n"]}),"\n",(0,i.jsx)(r.h2,{id:"coordinator-leader-election",children:"Coordinator Leader Election"}),"\n",(0,i.jsxs)(r.p,{children:["We use the Curator ",(0,i.jsx)(r.a,{href:"https://curator.apache.org/curator-recipes/leader-latch.html",children:"LeaderLatch"})," recipe to perform leader election at path"]}),"\n",(0,i.jsx)(r.pre,{children:(0,i.jsx)(r.code,{children:"${druid.zk.paths.coordinatorPath}/_COORDINATOR\n"})}),"\n",(0,i.jsx)(r.h2,{id:"segment-publishing-protocol-from-historical-and-realtime",children:'Segment "publishing" protocol from Historical and Realtime'}),"\n",(0,i.jsxs)(r.p,{children:["The ",(0,i.jsx)(r.code,{children:"announcementsPath"})," and ",(0,i.jsx)(r.code,{children:"liveSegmentsPath"})," are used for this."]}),"\n",(0,i.jsxs)(r.p,{children:["All ",(0,i.jsx)(r.a,{href:"/docs/33.0.0/design/historical",children:"Historical"})," processes publish themselves on the ",(0,i.jsx)(r.code,{children:"announcementsPath"}),", specifically, they will create an ephemeral znode at"]}),"\n",(0,i.jsx)(r.pre,{children:(0,i.jsx)(r.code,{children:"${druid.zk.paths.announcementsPath}/${druid.host}\n"})}),"\n",(0,i.jsx)(r.p,{children:"Which signifies that they exist. They will also subsequently create a permanent znode at"}),"\n",(0,i.jsx)(r.pre,{children:(0,i.jsx)(r.code,{children:"${druid.zk.paths.liveSegmentsPath}/${druid.host}\n"})}),"\n",(0,i.jsx)(r.p,{children:"And as they load up segments, they will attach ephemeral znodes that look like"}),"\n",(0,i.jsx)(r.pre,{children:(0,i.jsx)(r.code,{children:"${druid.zk.paths.liveSegmentsPath}/${druid.host}/_segment_identifier_\n"})}),"\n",(0,i.jsxs)(r.p,{children:["Processes like the ",(0,i.jsx)(r.a,{href:"/docs/33.0.0/design/coordinator",children:"Coordinator"})," and ",(0,i.jsx)(r.a,{href:"/docs/33.0.0/design/broker",children:"Broker"})," can then watch these paths to see which processes are currently serving which segments."]})]})}function h(e={}){const{wrapper:r}={...(0,s.R)(),...e.components};return r?(0,i.jsx)(r,{...e,children:(0,i.jsx)(l,{...e})}):l(e)}}}]); |