| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[879],{14137:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>k});var a=n(67294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?r(Object(n),!0).forEach((function(t){i(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function l(e,t){if(null==e)return{};var n,a,i=function(e,t){if(null==e)return{};var n,a,i={},r=Object.keys(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a<r.length;a++)n=r[a],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},m="mdxType",c={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,r=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),m=p(n),d=i,k=m["".concat(s,".").concat(d)]||m[d]||c[d]||r;return n?a.createElement(k,o(o({ref:t},u),{},{components:n})):a.createElement(k,o({ref:t},u))}));function k(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var r=n.length,o=new Array(r);o[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[m]="string"==typeof e?e:i,o[1]=l;for(var p=2;p<r;p++)o[p]=n[p];return a.createElement.apply(null,o)}return a.createElement.apply(null,n)}d.displayName="MDXCreateElement"},8737:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>s,default:()=>k,frontMatter:()=>l,metadata:()=>p,toc:()=>m});var a=n(87462),i=n(63366),r=(n(67294),n(14137)),o=["components"],l={id:"known-issues",title:"SQL-based ingestion known issues",sidebar_label:"Known issues"},s=void 0,p={unversionedId:"multi-stage-query/known-issues",id:"multi-stage-query/known-issues",title:"SQL-based ingestion known issues",description:"\x3c!--",source:"@site/docs/31.0.0/multi-stage-query/known-issues.md",sourceDirName:"multi-stage-query",slug:"/multi-stage-query/known-issues",permalink:"/docs/31.0.0/multi-stage-query/known-issues",draft:!1,tags:[],version:"current",frontMatter:{id:"known-issues",title:"SQL-based ingestion known issues",sidebar_label:"Known issues"},sidebar:"docs",previous:{title:"Reference",permalink:"/docs/31.0.0/multi-stage-query/reference"},next:{title:"Streaming ingestion",permalink:"/docs/31.0.0/ingestion/streaming"}},u={},m=[{value:"Multi-stage query task runtime",id:"multi-stage-query-task-runtime",level:2},{value:"<code>SELECT</code> Statement",id:"select-statement",level:2},{value:"<code>INSERT</code> and <code>REPLACE</code> Statements",id:"insert-and-replace-statements",level:2},{value:"<code>EXTERN</code> Function",id:"extern-function",level:2},{value:"<code>WINDOW</code> Function",id:"window-function",level:2},{value:"Automatic compaction",id:"automatic-compaction",level:2}],c={toc:m},d="wrapper";function k(e){var t=e.components,n=(0,i.Z)(e,o);return(0,r.kt)(d,(0,a.Z)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("admonition",{type:"info"},(0,r.kt)("p",{parentName:"admonition"}," This page describes SQL-based batch ingestion using the ",(0,r.kt)("a",{parentName:"p",href:"/docs/31.0.0/multi-stage-query/"},(0,r.kt)("inlineCode",{parentName:"a"},"druid-multi-stage-query")),"\nextension, new in Druid 24.0. Refer to the ",(0,r.kt)("a",{parentName:"p",href:"/docs/31.0.0/ingestion/#batch"},"ingestion methods")," table to determine which\ningestion method is right for you.")),(0,r.kt)("h2",{id:"multi-stage-query-task-runtime"},"Multi-stage query task runtime"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},"Fault tolerance is partially implemented. Workers get relaunched when they are killed unexpectedly. The controller does not get relaunched if it is killed unexpectedly.")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},"Worker task stage outputs are stored in the working directory given by ",(0,r.kt)("inlineCode",{parentName:"p"},"druid.indexer.task.baseDir"),". Stages that\ngenerate a large amount of output data may exhaust all available disk space. In this case, the query fails with\nan ",(0,r.kt)("a",{parentName:"p",href:"/docs/31.0.0/multi-stage-query/reference#error_UnknownError"},"UnknownError"),' with a message including "No space left on device".'))),(0,r.kt)("h2",{id:"select-statement"},(0,r.kt)("inlineCode",{parentName:"h2"},"SELECT")," Statement"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"GROUPING SETS")," are not implemented. Queries using these features return a\n",(0,r.kt)("a",{parentName:"li",href:"/docs/31.0.0/multi-stage-query/reference#error_QueryNotSupported"},"QueryNotSupported")," error.")),(0,r.kt)("h2",{id:"insert-and-replace-statements"},(0,r.kt)("inlineCode",{parentName:"h2"},"INSERT")," and ",(0,r.kt)("inlineCode",{parentName:"h2"},"REPLACE")," Statements"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},"The ",(0,r.kt)("inlineCode",{parentName:"p"},"INSERT")," and ",(0,r.kt)("inlineCode",{parentName:"p"},"REPLACE")," statements with column lists, like ",(0,r.kt)("inlineCode",{parentName:"p"},"INSERT INTO tbl (a, b, c) SELECT ..."),", is not implemented.")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"INSERT ... SELECT")," and ",(0,r.kt)("inlineCode",{parentName:"p"},"REPLACE ... SELECT")," insert columns from the ",(0,r.kt)("inlineCode",{parentName:"p"},"SELECT")," statement based on column name. This\ndiffers from SQL standard behavior, where columns are inserted based on position.")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"INSERT")," and ",(0,r.kt)("inlineCode",{parentName:"p"},"REPLACE")," do not support all options available in ",(0,r.kt)("a",{parentName:"p",href:"/docs/31.0.0/ingestion/ingestion-spec"},"ingestion specs"),",\nincluding the ",(0,r.kt)("inlineCode",{parentName:"p"},"createBitmapIndex")," and ",(0,r.kt)("inlineCode",{parentName:"p"},"multiValueHandling")," ",(0,r.kt)("a",{parentName:"p",href:"/docs/31.0.0/ingestion/ingestion-spec#dimension-objects"},"dimension"),"\nproperties, and the ",(0,r.kt)("inlineCode",{parentName:"p"},"indexSpec")," ",(0,r.kt)("a",{parentName:"p",href:"/docs/31.0.0/ingestion/ingestion-spec#tuningconfig"},(0,r.kt)("inlineCode",{parentName:"a"},"tuningConfig"))," property."))),(0,r.kt)("h2",{id:"extern-function"},(0,r.kt)("inlineCode",{parentName:"h2"},"EXTERN")," Function"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},"The ",(0,r.kt)("a",{parentName:"p",href:"/docs/31.0.0/ingestion/ingestion-spec#inclusions-and-exclusions"},"schemaless dimensions"),"\nfeature is not available. All columns and their types must be specified explicitly using the ",(0,r.kt)("inlineCode",{parentName:"p"},"signature")," parameter\nof the ",(0,r.kt)("a",{parentName:"p",href:"/docs/31.0.0/multi-stage-query/reference#extern-function"},(0,r.kt)("inlineCode",{parentName:"a"},"EXTERN")," function"),".")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"EXTERN")," with input sources that match large numbers of files may exhaust available memory on the controller task.")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("p",{parentName:"li"},(0,r.kt)("inlineCode",{parentName:"p"},"EXTERN")," refers to external files. Use ",(0,r.kt)("inlineCode",{parentName:"p"},"FROM")," to access ",(0,r.kt)("inlineCode",{parentName:"p"},"druid")," input sources."))),(0,r.kt)("h2",{id:"window-function"},(0,r.kt)("inlineCode",{parentName:"h2"},"WINDOW")," Function"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"The maximum number of elements in a window cannot exceed a value of 100,000. "),(0,r.kt)("li",{parentName:"ul"},"To avoid ",(0,r.kt)("inlineCode",{parentName:"li"},"leafOperators")," in MSQ engine, window functions have an extra scan stage after the window stage for cases\nwhere native engine has a non-empty ",(0,r.kt)("inlineCode",{parentName:"li"},"leafOperator"),".")),(0,r.kt)("h2",{id:"automatic-compaction"},"Automatic compaction"),(0,r.kt)("p",null,"The following known issues and limitations affect automatic compaction with the MSQ task engine:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"The ",(0,r.kt)("inlineCode",{parentName:"li"},"metricSpec")," field is only supported for certain aggregators. For more information, see ",(0,r.kt)("a",{parentName:"li",href:"/docs/31.0.0/data-management/automatic-compaction#supported-aggregators"},"Supported aggregators"),"."),(0,r.kt)("li",{parentName:"ul"},"Only dynamic and range-based partitioning are supported."),(0,r.kt)("li",{parentName:"ul"},"Set ",(0,r.kt)("inlineCode",{parentName:"li"},"rollup")," to ",(0,r.kt)("inlineCode",{parentName:"li"},"true")," if and only if ",(0,r.kt)("inlineCode",{parentName:"li"},"metricSpec")," is not empty or null."),(0,r.kt)("li",{parentName:"ul"},"You can only partition on string dimensions. However, multi-valued string dimensions are not supported."),(0,r.kt)("li",{parentName:"ul"},"The ",(0,r.kt)("inlineCode",{parentName:"li"},"maxTotalRows")," config is not supported in ",(0,r.kt)("inlineCode",{parentName:"li"},"DynamicPartitionsSpec"),". Use ",(0,r.kt)("inlineCode",{parentName:"li"},"maxRowsPerSegment")," instead."),(0,r.kt)("li",{parentName:"ul"},"Segments can only be sorted on ",(0,r.kt)("inlineCode",{parentName:"li"},"__time")," as the first column.")))}k.isMDXComponent=!0}}]); |