Google Git
Sign in
apache / pinot-site / 3f5c935562a17c589ffbeee812589921860ab707 / . / assets / js / a176cc64.2345cad2.js
blob: a3c8d8ea5638d7d5f9212a79799f9123d9d82923 [file] [log] [blame]
"use strict";(self.webpackChunkwebsite=self.webpackChunkwebsite||[]).push([[8935],{4137:function(e,t,a){a.d(t,{Zo:function(){return p},kt:function(){return c}});var n=a(7294);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t<arguments.length;t++){var a=null!=arguments[t]?arguments[t]:{};t%2?l(Object(a),!0).forEach((function(t){r(e,t,a[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):l(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))}))}return e}function o(e,t){if(null==e)return{};var a,n,r=function(e,t){if(null==e)return{};var a,n,r={},l=Object.keys(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n<l.length;n++)a=l[n],t.indexOf(a)>=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=n.createContext({}),u=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},p=function(e){var t=u(e.components);return n.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),m=u(a),c=r,h=m["".concat(s,".").concat(c)]||m[c]||d[c]||l;return a?n.createElement(h,i(i({ref:t},p),{},{components:a})):n.createElement(h,i({ref:t},p))}));function c(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=a.length,i=new Array(l);i[0]=m;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o.mdxType="string"==typeof e?e:r,i[1]=o;for(var u=2;u<l;u++)i[u]=a[u];return n.createElement.apply(null,i)}return n.createElement.apply(null,a)}m.displayName="MDXCreateElement"},9985:function(e,t,a){a.r(t),a.d(t,{contentTitle:function(){return s},default:function(){return m},frontMatter:function(){return o},metadata:function(){return u},toc:function(){return p}});var n=a(7462),r=a(3366),l=(a(7294),a(4137)),i=["components"],o={title:"Announcing Apache Pinot 0.10",author:"Apache Pinot Engineering Team",author_title:"Apache Pinot Engineering Team",author_url:"https://twitter.com/ApachePinot",author_image_url:"https://pinot.apache.org/authors/pinot_team.jpg",description:"Learn more about the release of Apache Pinot 0.10 and all of new features that have been included in this version of the product.",keywords:["Apache Pinot","Apache Pinot Releases","Apache Pinot 0.10.0"],tags:["Pinot","Data","Analytics","User-Facing Analytics","Releases"]},s=void 0,u={permalink:"/blog/2022/04/04/Announcing-Apache-Pinot-0-10",editUrl:"https://github.com/apache/pinot-site/edit/dev/website/blog/2022-04-04-Announcing-Apache-Pinot-0-10.md",source:"@site/blog/2022-04-04-Announcing-Apache-Pinot-0-10.md",title:"Announcing Apache Pinot 0.10",description:"Learn more about the release of Apache Pinot 0.10 and all of new features that have been included in this version of the product.",date:"2022-04-04T00:00:00.000Z",formattedDate:"April 4, 2022",tags:[{label:"Pinot",permalink:"/blog/tags/pinot"},{label:"Data",permalink:"/blog/tags/data"},{label:"Analytics",permalink:"/blog/tags/analytics"},{label:"User-Facing Analytics",permalink:"/blog/tags/user-facing-analytics"},{label:"Releases",permalink:"/blog/tags/releases"}],readingTime:4.615,truncated:!1,nextItem:{title:"Text analytics on LinkedIn Talent Insights using Apache Pinot",permalink:"/blog/2021/06/16/LinkedIn-TextAnalytics"}},p=[{value:"Query Plans",id:"query-plans",children:[]},{value:"FILTER Clauses for Aggregates",id:"filter-clauses-for-aggregates",children:[]},{value:"greatest and least",id:"greatest-and-least",children:[]},{value:"DistinctCountSmartHLL",id:"distinctcountsmarthll",children:[]},{value:"UI updates",id:"ui-updates",children:[]},{value:"RealTimeToOffline Task",id:"realtimetooffline-task",children:[]},{value:"Empty QuickStart",id:"empty-quickstart",children:[]},{value:"Data Ingestion",id:"data-ingestion",children:[]},{value:"Other changes",id:"other-changes",children:[]},{value:"Dependency updates",id:"dependency-updates",children:[]},{value:"Resources",id:"resources",children:[]}],d={toc:p};function m(e){var t=e.components,o=(0,r.Z)(e,i);return(0,l.kt)("wrapper",(0,n.Z)({},d,o,{components:t,mdxType:"MDXLayout"}),(0,l.kt)("p",null,"We are excited to announce the release this week of Apache Pinot 0.10.\nApache Pinot is a real-time distributed datastore designed to answer OLAP queries with high throughput and low latency."),(0,l.kt)("p",null,"This release is cut from commit ",(0,l.kt)("a",{parentName:"p",href:"https://github.com/apache/pinot/commit/fd9c58a11ed16d27109baefcee138eea30132ad3"},"fd9c58a11ed16d27109baefcee138eea30132ad3"),".\nYou can find a full list of everything included in the ",(0,l.kt)("a",{parentName:"p",href:"https://docs.pinot.apache.org/basics/releases/0.10.0"},"release notes"),"."),(0,l.kt)("p",null,"Let\u2019s have a look at some of the changes, with the help of the batch ",(0,l.kt)("a",{parentName:"p",href:"https://docs.pinot.apache.org/basics/getting-started/running-pinot-in-docker"},"QuickStart configuration"),"."),(0,l.kt)("h2",{id:"query-plans"},"Query Plans"),(0,l.kt)("p",null,"Amrish Lal implemented the ",(0,l.kt)("inlineCode",{parentName:"p"},"EXPLAIN PLAN")," clause, which returns the execution plan that will be chosen by the Pinot Query Engine.\nThis lets us see what the query is likely to do without actually having to run it."),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-sql"},"EXPLAIN PLAN FOR\nSELECT *\nFROM baseballStats\nWHERE league = 'NL'\n")),(0,l.kt)("p",null,"If we run this query, we'll see the following results:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Operator"),(0,l.kt)("th",{parentName:"tr",align:null},"Operator_Id"),(0,l.kt)("th",{parentName:"tr",align:null},"Parent_Id"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"BROKER_REDUCE(limit:10)"),(0,l.kt)("td",{parentName:"tr",align:null},"0"),(0,l.kt)("td",{parentName:"tr",align:null},"-1")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"COMBINE_SELECT"),(0,l.kt)("td",{parentName:"tr",align:null},"1"),(0,l.kt)("td",{parentName:"tr",align:null},"0")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"SELECT(selectList:AtBatting, G_old, baseOnBalls, caughtStealing, doules, groundedIntoDoublePlays, hits, hitsByPitch, homeRuns, intentionalWalks, league, numberOfGames, numberOfGamesAsBatter, playerID, playerName, playerStint, runs, runsBattedIn, sacrificeFlies, sacrificeHits, stolenBases, strikeouts, teamID, tripples, yearID)"),(0,l.kt)("td",{parentName:"tr",align:null},"2"),(0,l.kt)("td",{parentName:"tr",align:null},"1")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"TRANSFORM_PASSTHROUGH(AtBatting, G_old, baseOnBalls, caughtStealing, doules, groundedIntoDoublePlays, hits, hitsByPitch, homeRuns, intentionalWalks, league, numberOfGames, numberOfGamesAsBatter, playerID, playerName, playerStint, runs, runsBattedIn, sacrificeFlies, sacrificeHits, stolenBases, strikeouts, teamID, tripples, yearID)"),(0,l.kt)("td",{parentName:"tr",align:null},"3"),(0,l.kt)("td",{parentName:"tr",align:null},"2")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"PROJECT(homeRuns, playerStint, groundedIntoDoublePlays, numberOfGames, AtBatting, stolenBases, tripples, hitsByPitch, teamID, numberOfGamesAsBatter, strikeouts, sacrificeFlies, caughtStealing, baseOnBalls, playerName, doules, league, yearID, hits, runsBattedIn, G_old, sacrificeHits, intentionalWalks, runs, playerID)"),(0,l.kt)("td",{parentName:"tr",align:null},"4"),(0,l.kt)("td",{parentName:"tr",align:null},"3")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"FILTER_FULL_SCAN(operator:EQ,predicate:league = 'NL')"),(0,l.kt)("td",{parentName:"tr",align:null},"5"),(0,l.kt)("td",{parentName:"tr",align:null},"4")))),(0,l.kt)("h2",{id:"filter-clauses-for-aggregates"},"FILTER Clauses for Aggregates"),(0,l.kt)("p",null,"Atri Sharma added the filter clause for aggregates.\nThis feature makes it possible to write queries like this:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT SUM(homeRuns) FILTER(WHERE league = 'NL') AS nlHomeRuns,\n SUM(homeRuns) FILTER(WHERE league = 'AL') AS alHomeRuns\nFROM baseballStats\n")),(0,l.kt)("p",null,"If we run this query, we'll see the following output:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"nlHomeRuns"),(0,l.kt)("th",{parentName:"tr",align:null},"alHomeRuns"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"135486"),(0,l.kt)("td",{parentName:"tr",align:null},"135990")))),(0,l.kt)("h2",{id:"greatest-and-least"},"greatest and least"),(0,l.kt)("p",null,"Richard Startin added the ",(0,l.kt)("inlineCode",{parentName:"p"},"greatest")," and ",(0,l.kt)("inlineCode",{parentName:"p"},"least")," functions:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT playerID,\n least(5.0, max(homeRuns)) AS homeRuns,\n greatest(5.0, max(hits)) AS hits\nFROM baseballStats\nWHERE league = 'NL' AND teamID = 'SFN'\nGROUP BY playerID\nLIMIT 5\n")),(0,l.kt)("p",null,"If we run this query, we'll see the following output:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"playerID"),(0,l.kt)("th",{parentName:"tr",align:null},"homeRuns"),(0,l.kt)("th",{parentName:"tr",align:null},"hits"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"ramirju01"),(0,l.kt)("td",{parentName:"tr",align:null},"0"),(0,l.kt)("td",{parentName:"tr",align:null},"5")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"milneed01"),(0,l.kt)("td",{parentName:"tr",align:null},"4"),(0,l.kt)("td",{parentName:"tr",align:null},"54")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"testani01"),(0,l.kt)("td",{parentName:"tr",align:null},"0"),(0,l.kt)("td",{parentName:"tr",align:null},"5")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"shawbo01"),(0,l.kt)("td",{parentName:"tr",align:null},"0"),(0,l.kt)("td",{parentName:"tr",align:null},"8")),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"vogelry01"),(0,l.kt)("td",{parentName:"tr",align:null},"0"),(0,l.kt)("td",{parentName:"tr",align:null},"12")))),(0,l.kt)("h2",{id:"distinctcountsmarthll"},"DistinctCountSmartHLL"),(0,l.kt)("p",null," Xiaotian (Jackie) Jiang added the ",(0,l.kt)("inlineCode",{parentName:"p"},"DistinctCountSmartHLL")," aggregation function, which automatically converts the Set to HyperLogLog if the set size grows too big to protect the servers from running out of memory:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT DISTINCTCOUNTSMARTHLL(homeRuns, 'hllLog2m=8;hllConversionThreshold=10')\nFROM baseballStats\n")),(0,l.kt)("p",null,"If we run this query, we'll see the following output:"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"distinctcountsmarthll(homeRuns)"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"66")))),(0,l.kt)("h2",{id:"ui-updates"},"UI updates"),(0,l.kt)("p",null,"There were also a bunch of updates to the Pinot Data Explorer, by Sanket Shah and Johan Adami."),(0,l.kt)("p",null,"The display of reported size and estimated size is now in a human readable format:"),(0,l.kt)("p",null,(0,l.kt)("img",{alt:"Human readable sizes",src:a(2019).Z})),(0,l.kt)("p",null,"Fixes for the following issues:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"Error messages weren't showing on the UI when an invalid operation is attempted:")),(0,l.kt)("p",null,(0,l.kt)("img",{alt:"A backwards incompatible attempted schema change",src:a(734).Z})),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"Query console goes blank on syntax error."),(0,l.kt)("li",{parentName:"ul"},"Query console cannot show query result when multiple columns have the same name."),(0,l.kt)("li",{parentName:"ul"},"Adding extra fields after ",(0,l.kt)("inlineCode",{parentName:"li"},"SELECT *")," would throw a NullPointerException."),(0,l.kt)("li",{parentName:"ul"},"Some queries were returning ",(0,l.kt)("inlineCode",{parentName:"li"},"--")," instead of ",(0,l.kt)("inlineCode",{parentName:"li"},"0"),"."),(0,l.kt)("li",{parentName:"ul"},"Query console couldn't show the query result if multiple columns had the same name."),(0,l.kt)("li",{parentName:"ul"},"Pinot Dashboard tenant view showing the incorrect amount of servers and brokers.")),(0,l.kt)("h2",{id:"realtimetooffline-task"},"RealTimeToOffline Task"),(0,l.kt)("p",null,"Xiaotian (Jackie) Jiang made some fixes to the ",(0,l.kt)("a",{parentName:"p",href:"https://dev.startree.ai/docs/pinot/recipes/real-time-offline-job"},"RealTimeToOffline job")," to handle time gaps and proceed to the next time window when no segment matches the current one."),(0,l.kt)("h2",{id:"empty-quickstart"},"Empty QuickStart"),(0,l.kt)("p",null,"Kenny Bastani added an empty QuickStart command, which lets you quickly spin up an empty Pinot cluster:"),(0,l.kt)("pre",null,(0,l.kt)("code",{parentName:"pre",className:"language-bash"},"docker run \\\n -p 8000:8000 \\\n -p 9000:9000 \\\n apachepinot/pinot:0.10.0 QuickStart \\\n -type empty\n")),(0,l.kt)("p",null,"You can then ingest your own dataset without needing to worry about spinning up each of the Pinot components individually."),(0,l.kt)("h2",{id:"data-ingestion"},"Data Ingestion"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Richard Startin fixed some issues with real-time ingestion where consumption of messages would stop if a bad batch of messages was consumed from Kafka.")),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Mohemmad Zaid Khan added the BoundedColumnValue partition function, which partitions segments based on column values.")),(0,l.kt)("li",{parentName:"ul"},(0,l.kt)("p",{parentName:"li"},"Xiaobing Li added the fixed name segment generator, which can be used when you want to replace a specific existing segment."))),(0,l.kt)("h2",{id:"other-changes"},"Other changes"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"Richard Startin set LZ4 compression as the default for all metrics fields."),(0,l.kt)("li",{parentName:"ul"},"Mark Needham added the ",(0,l.kt)("inlineCode",{parentName:"li"},"ST_Within")," geospatial function."),(0,l.kt)("li",{parentName:"ul"},"Rong Rong fixed a bug where query stats wouldn't show if there was an error processing the query (e.g. if the query timed out)."),(0,l.kt)("li",{parentName:"ul"},"Prashant Pandey fixed the query engine to handle extra columns added to a ",(0,l.kt)("inlineCode",{parentName:"li"},"SELECT *")," statement."),(0,l.kt)("li",{parentName:"ul"},"Richard Startin added support for forward indexes on JSON columns."),(0,l.kt)("li",{parentName:"ul"},"Rong Rong added the GRPC broker request handler so that data can be streamed back from the server to the broker when processing queries."),(0,l.kt)("li",{parentName:"ul"},"deemoliu made it possible to add a default strategy when using the ",(0,l.kt)("a",{parentName:"li",href:"https://dev.startree.ai/docs/pinot/recipes/upserts-partial"},"partial upsert feature"),"."),(0,l.kt)("li",{parentName:"ul"},"Jeff Moszuti added support for the ",(0,l.kt)("inlineCode",{parentName:"li"},"TIMESTAMP")," data type in the ",(0,l.kt)("a",{parentName:"li",href:"https://docs.pinot.apache.org/operators/configuration-recommendation-engine"},"configuration recommendation engine"),".")),(0,l.kt)("h2",{id:"dependency-updates"},"Dependency updates"),(0,l.kt)("p",null,"The following dependencies were updated:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"async-http-client because the library moved to a different organization."),(0,l.kt)("li",{parentName:"ul"},"RoaringBitmap to 0.9.25"),(0,l.kt)("li",{parentName:"ul"},"JsonPath to 2.7.0"),(0,l.kt)("li",{parentName:"ul"},"Kafka to 2.8.1"),(0,l.kt)("li",{parentName:"ul"},"Prometheus to 0.16.1")),(0,l.kt)("h2",{id:"resources"},"Resources"),(0,l.kt)("p",null,"If you want to try out Apache Pinot, the following resources will help you get started:"),(0,l.kt)("ul",null,(0,l.kt)("li",{parentName:"ul"},"Download page: ",(0,l.kt)("a",{parentName:"li",href:"https://pinot.apache.org/download/"},"https://pinot.apache.org/download/")),(0,l.kt)("li",{parentName:"ul"},"Getting started: ",(0,l.kt)("a",{parentName:"li",href:"https://docs.pinot.apache.org/getting-started"},"https://docs.pinot.apache.org/getting-started")),(0,l.kt)("li",{parentName:"ul"},"Apache Pinot Recipes: ",(0,l.kt)("a",{parentName:"li",href:"https://dev.startree.ai/docs/pinot/recipes/"},"https://dev.startree.ai/docs/pinot/recipes/")),(0,l.kt)("li",{parentName:"ul"},"Join our Slack channel: ",(0,l.kt)("a",{parentName:"li",href:"https://communityinviter.com/apps/apache-pinot/apache-pinot"},"https://communityinviter.com/apps/apache-pinot/apache-pinot")),(0,l.kt)("li",{parentName:"ul"},"See our upcoming events: ",(0,l.kt)("a",{parentName:"li",href:"https://www.meetup.com/apache-pinot"},"https://www.meetup.com/apache-pinot")),(0,l.kt)("li",{parentName:"ul"},"Follow us on Twitter: ",(0,l.kt)("a",{parentName:"li",href:"https://twitter.com/startreedata"},"https://twitter.com/startreedata")),(0,l.kt)("li",{parentName:"ul"},"Subscribe to our YouTube channel: ",(0,l.kt)("a",{parentName:"li",href:"https://www.youtube.com/c/StarTree"},"https://www.youtube.com/c/StarTree"))))}m.isMDXComponent=!0},734:function(e,t,a){t.Z=a.p+"assets/images/backwards-incompatible-99886dcd0be55a8100a7d6c5f3da3bda.png"},2019:function(e,t,a){t.Z=a.p+"assets/images/human-readable-sizes-b8c4009dd53d23da3b8637963827a8de.png"}}]);