| "use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[5348],{4137:function(e,t,n){n.d(t,{Zo:function(){return p},kt:function(){return h}});var o=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,o,a=function(e,t){if(null==e)return{};var n,o,a={},r=Object.keys(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o<r.length;o++)n=r[o],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=o.createContext({}),l=function(e){var t=o.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=l(e.components);return o.createElement(c.Provider,{value:t},e.children)},m="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},d=o.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,c=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),m=l(n),d=a,h=m["".concat(c,".").concat(d)]||m[d]||u[d]||r;return n?o.createElement(h,i(i({ref:t},p),{},{components:n})):o.createElement(h,i({ref:t},p))}));function h(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=d;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s[m]="string"==typeof e?e:a,i[1]=s;for(var l=2;l<r;l++)i[l]=n[l];return o.createElement.apply(null,i)}return o.createElement.apply(null,n)}d.displayName="MDXCreateElement"},2176:function(e,t,n){n.r(t),n.d(t,{contentTitle:function(){return c},default:function(){return d},frontMatter:function(){return s},metadata:function(){return l},toc:function(){return p}});var o=n(7462),a=n(3366),r=(n(7294),n(4137)),i=["components"],s={title:"Segment Compaction for Upsert Enabled Tables in Apache Pinot",author:"Robert Zych",author_title:"Software Engineer",author_url:"https://www.linkedin.com/in/robertzych/",author_image_url:"https://pinot.apache.org/authors/pinot_team.jpg",description:"The blog post discusses the feature contribution of segment compaction to Apache Pinot project, addressing the issue of older records consuming unnecessary storage space. It explains the configuration and impact of segment compaction in freeing up storage. The author expresses gratitude and offers support for questions or feedback on segment compaction.",keywords:["Apache Pinot","blog post","feature contribution","segment compaction","Apache Pinot project","older records","storage space","freeing up storage"],tags:["Pinot","Data","Analytics","User-Facing Analytics","blog post","feature contribution","segment compaction","Apache Pinot project","older records","storage space","configuration","impact","freeing up storage"]},c=void 0,l={permalink:"/blog/2023/08/04/segment-compaction-for-upsert-enabled-tables-in-apache-pinot-3f30657aa077",editUrl:"https://github.com/apache/pinot-site/edit/dev/website/blog/2023-08-04-segment-compaction-for-upsert-enabled-tables-in-apache-pinot-3f30657aa077.md",source:"@site/blog/2023-08-04-segment-compaction-for-upsert-enabled-tables-in-apache-pinot-3f30657aa077.md",title:"Segment Compaction for Upsert Enabled Tables in Apache Pinot",description:"The blog post discusses the feature contribution of segment compaction to Apache Pinot project, addressing the issue of older records consuming unnecessary storage space. It explains the configuration and impact of segment compaction in freeing up storage. The author expresses gratitude and offers support for questions or feedback on segment compaction.",date:"2023-08-04T00:00:00.000Z",formattedDate:"August 4, 2023",tags:[{label:"Pinot",permalink:"/blog/tags/pinot"},{label:"Data",permalink:"/blog/tags/data"},{label:"Analytics",permalink:"/blog/tags/analytics"},{label:"User-Facing Analytics",permalink:"/blog/tags/user-facing-analytics"},{label:"blog post",permalink:"/blog/tags/blog-post"},{label:"feature contribution",permalink:"/blog/tags/feature-contribution"},{label:"segment compaction",permalink:"/blog/tags/segment-compaction"},{label:"Apache Pinot project",permalink:"/blog/tags/apache-pinot-project"},{label:"older records",permalink:"/blog/tags/older-records"},{label:"storage space",permalink:"/blog/tags/storage-space"},{label:"configuration",permalink:"/blog/tags/configuration"},{label:"impact",permalink:"/blog/tags/impact"},{label:"freeing up storage",permalink:"/blog/tags/freeing-up-storage"}],readingTime:3.565,truncated:!1,prevItem:{title:"Announcing Apache Pinot 1.0\u2122",permalink:"/blog/2023/09/19/Annoucing-Apache-Pinot-1-0"},nextItem:{title:"Star-Tree Index in Apache Pinot - Part 3 - Understanding the Impact in Real Customer Scenarios",permalink:"/blog/2023/07/12/star-tree-index-in-apache-pinot-part-3-understanding-the-impact-in-real-customer"}},p=[{value:"Context and Configuration",id:"context-and-configuration",children:[]},{value:"Example Use Case",id:"example-use-case",children:[]},{value:"Conclusion",id:"conclusion",children:[]}],m={toc:p},u="wrapper";function d(e){var t=e.components,n=(0,a.Z)(e,i);return(0,r.kt)(u,(0,o.Z)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("p",null,"I\u2019m happy to share that my 1st feature contribution to the Apache Pinot project (",(0,r.kt)("a",{parentName:"p",href:"https://github.com/apache/pinot/pull/10463"},"Segment compaction for upsert enabled real-time tables"),") was merged recently! In this post, I will briefly discuss the problem segment compaction addresses, how to configure it, and what it looks like in action. If you\u2019re unfamiliar with Pinot\u2019s Upsert features, I recommend reviewing ",(0,r.kt)("a",{parentName:"p",href:"https://dev.startree.ai/docs/pinot/recipes/upserts-full"},"Full Upserts in Pinot")," to get started and ",(0,r.kt)("a",{parentName:"p",href:"https://docs.pinot.apache.org/basics/data-import/upsert"},"Stream Ingestion with Upsert")," for more information."),(0,r.kt)("h2",{id:"context-and-configuration"},"Context and Configuration"),(0,r.kt)("p",null,"As Pinot\u2019s Upsert stores all versions of the record ingested into immutable segments on disk, older records unnecessarily consume valuable storage space when they\u2019re no longer used in query results. Pinot\u2019s Segment Compaction reclaims this valuable storage space by introducing a periodic process that replaces the completed segments with compacted segments which only contain the latest version of the records. I recommend reviewing the Minion documentation if you\u2019re unfamiliar with Pinot\u2019s ability to run periodic processes."),(0,r.kt)("p",null,"With task scheduling enabled and an available Minion, you can configure segment compaction by adding the following to your table\u2019s config."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-json"},'"task": {\n "taskTypeConfigsMap": {\n "UpsertCompactionTask": {\n "schedule": "0 */5 * ? * *",\n "bufferTimePeriod": "7d",\n "invalidRecordsThresholdPercent": "30",\n "invalidRecordsThresholdCount": "100000"\n }\n }\n}\n')),(0,r.kt)("p",null,"All the configs above (excluding schedule) determine which completed segments are selected for compaction."),(0,r.kt)("p",null,"bufferTimePeriod is the amount of time that has elapsed since the segment was consuming. In the example above, this has been set to \u201c7d\u201d which means that any segment that was completed over 7 days ago may be eligible for compaction. However, if you want to ensure that segments are compacted without any additional delay this config can be set to \u201c0d\u201d."),(0,r.kt)("p",null,"invalidRecordsThresholdPercent is a limit to the amount of older records allowed in the completed segment represented as a percentage of the total number of records in the segment (i.e. old records / total records). In the example above, this has been set to \u201c30\u201d which means that if more than 30% of the records in the completed segment are old, then the segment may be selected for compaction. As segment compaction is an expensive operation, it is not recommended to set this config (or invalidRecordsThresholdCount) too close to 1. This config is optional on the condition that invalidRecordsThresholdCount has been set and can be used in conjunction with invalidRecordsThresholdCount."),(0,r.kt)("p",null,"invalidRecordsThresholdCount is also a limit similar to invalidRecordsThresholdPercent, but allows you to express the threshold as a record count. In the example above, this has been set to \u201c100000\u201d which means that if the segment contains more than 100K records then it may be selected for compaction."),(0,r.kt)("h2",{id:"example-use-case"},"Example Use Case"),(0,r.kt)("p",null,"I\u2019ve created a data set that includes 24M records. The data set contains 240K unique keys that have each been duplicated 100 times."),(0,r.kt)("p",null,(0,r.kt)("img",{parentName:"p",src:"https://miro.medium.com/v2/resize:fit:4800/0*gEyUq_Tp4Fycgp2r",alt:"alt"})),(0,r.kt)("p",null,"After ingesting the data set there are 6 segments (5 completed segments + 1 consuming segment) with a total estimated size of 22.8MB. Submitting the query \u201cset skipUpsert=true; select count(","*",") from transcript","_","upsert\u201d before compaction produces the following query result."),(0,r.kt)("p",null,(0,r.kt)("img",{parentName:"p",src:"https://miro.medium.com/v2/resize:fit:1064/0*GE3P1fqAcsr0Xs5A",alt:"alt"})),(0,r.kt)("p",null,"After the compaction tasks are complete, the Minion Task Manager UI reports the following."),(0,r.kt)("p",null,(0,r.kt)("img",{parentName:"p",src:"https://miro.medium.com/v2/resize:fit:2000/0*SMxDZNndFwpoeNMI",alt:"alt"})),(0,r.kt)("p",null,"Segment compaction generates a task for each segment to be compacted. 5 tasks were generated in this case because 90% of the records (3.6\u20134.5M records) are old in all 5 of the completed segments, therefore exceeding the configured thresholds. If a completed segment only contains old records, it is deleted immediately and a task isn\u2019t generated to compact it."),(0,r.kt)("p",null,(0,r.kt)("img",{parentName:"p",src:"https://miro.medium.com/v2/resize:fit:1068/0*LB1itt-wCohpz42i",alt:"alt"})),(0,r.kt)("p",null,"Submitting the query again we now see that count matches the set of 240K unique keys."),(0,r.kt)("p",null,(0,r.kt)("img",{parentName:"p",src:"https://miro.medium.com/v2/resize:fit:2000/0*cmx4zxoMsD4-tR_u",alt:"alt"})),(0,r.kt)("p",null,"Once compaction has completed and the original segments have been replaced with their compacted counterparts we see that the total number of segments remained the same, but the total estimated size dropped to only 2.77MB! Since compaction can yield very small segments, one improvement would be to merge smaller segments into larger ones as this would improve query latency."),(0,r.kt)("h2",{id:"conclusion"},"Conclusion"),(0,r.kt)("p",null,"In this brief overview of Segment Compaction I covered the problem it addresses, how you can configure it, and demonstrated its ability to reclaim storage space. I\u2019d like to thank Ankit Sultana, Seunghyun Lee, and especially Jackie Jiang for their feedback and support throughout the design and development stages. If you have any questions or feedback, I\u2019m available on the Apache Pinot Slack."))}d.isMDXComponent=!0}}]); |