| "use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8890],{15680:(e,n,t)=>{t.d(n,{xA:()=>u,yg:()=>g});var r=t(96540);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function o(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?i(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):i(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function s(e,n){if(null==e)return{};var t,r,a=function(e,n){if(null==e)return{};var t,r,a={},i=Object.keys(e);for(r=0;r<i.length;r++)t=i[r],n.indexOf(t)>=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r<i.length;r++)t=i[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var l=r.createContext({}),p=function(e){var n=r.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):o(o({},n),e)),t},u=function(e){var n=p(e.components);return r.createElement(l.Provider,{value:n},e.children)},m="mdxType",c={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},d=r.forwardRef((function(e,n){var t=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),m=p(t),d=a,g=m["".concat(l,".").concat(d)]||m[d]||c[d]||i;return t?r.createElement(g,o(o({ref:n},u),{},{components:t})):r.createElement(g,o({ref:n},u))}));function g(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var i=t.length,o=new Array(i);o[0]=d;var s={};for(var l in n)hasOwnProperty.call(n,l)&&(s[l]=n[l]);s.originalType=e,s[m]="string"==typeof e?e:a,o[1]=s;for(var p=2;p<i;p++)o[p]=t[p];return r.createElement.apply(null,o)}return r.createElement.apply(null,t)}d.displayName="MDXCreateElement"},84698:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>u,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>m});var r=t(58168),a=t(98587),i=(t(96540),t(15680)),o=["components"],s={id:"known-issues",title:"SQL-based ingestion known issues",sidebar_label:"Known issues"},l=void 0,p={unversionedId:"multi-stage-query/known-issues",id:"multi-stage-query/known-issues",title:"SQL-based ingestion known issues",description:"\x3c!--",source:"@site/docs/29.0.0/multi-stage-query/known-issues.md",sourceDirName:"multi-stage-query",slug:"/multi-stage-query/known-issues",permalink:"/docs/29.0.0/multi-stage-query/known-issues",draft:!1,tags:[],version:"current",frontMatter:{id:"known-issues",title:"SQL-based ingestion known issues",sidebar_label:"Known issues"},sidebar:"docs",previous:{title:"Reference",permalink:"/docs/29.0.0/multi-stage-query/reference"},next:{title:"Streaming ingestion",permalink:"/docs/29.0.0/ingestion/streaming"}},u={},m=[{value:"Multi-stage query task runtime",id:"multi-stage-query-task-runtime",level:2},{value:"<code>SELECT</code> Statement",id:"select-statement",level:2},{value:"<code>INSERT</code> and <code>REPLACE</code> Statements",id:"insert-and-replace-statements",level:2},{value:"<code>EXTERN</code> Function",id:"extern-function",level:2}],c={toc:m},d="wrapper";function g(e){var n=e.components,t=(0,a.A)(e,o);return(0,i.yg)(d,(0,r.A)({},c,t,{components:n,mdxType:"MDXLayout"}),(0,i.yg)("admonition",{type:"info"},(0,i.yg)("p",{parentName:"admonition"}," This page describes SQL-based batch ingestion using the ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/multi-stage-query/"},(0,i.yg)("inlineCode",{parentName:"a"},"druid-multi-stage-query")),"\nextension, new in Druid 24.0. Refer to the ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/ingestion/#batch"},"ingestion methods")," table to determine which\ningestion method is right for you.")),(0,i.yg)("h2",{id:"multi-stage-query-task-runtime"},"Multi-stage query task runtime"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"Fault tolerance is partially implemented. Workers get relaunched when they are killed unexpectedly. The controller does not get relaunched if it is killed unexpectedly.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"Worker task stage outputs are stored in the working directory given by ",(0,i.yg)("inlineCode",{parentName:"p"},"druid.indexer.task.baseDir"),". Stages that\ngenerate a large amount of output data may exhaust all available disk space. In this case, the query fails with\nan ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/multi-stage-query/reference#error_UnknownError"},"UnknownError"),' with a message including "No space left on device".'))),(0,i.yg)("h2",{id:"select-statement"},(0,i.yg)("inlineCode",{parentName:"h2"},"SELECT")," Statement"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"GROUPING SETS")," are not implemented. Queries using these features return a\n",(0,i.yg)("a",{parentName:"li",href:"/docs/29.0.0/multi-stage-query/reference#error_QueryNotSupported"},"QueryNotSupported")," error.")),(0,i.yg)("h2",{id:"insert-and-replace-statements"},(0,i.yg)("inlineCode",{parentName:"h2"},"INSERT")," and ",(0,i.yg)("inlineCode",{parentName:"h2"},"REPLACE")," Statements"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"The ",(0,i.yg)("inlineCode",{parentName:"p"},"INSERT")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"REPLACE")," statements with column lists, like ",(0,i.yg)("inlineCode",{parentName:"p"},"INSERT INTO tbl (a, b, c) SELECT ..."),", is not implemented.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},(0,i.yg)("inlineCode",{parentName:"p"},"INSERT ... SELECT")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"REPLACE ... SELECT")," insert columns from the ",(0,i.yg)("inlineCode",{parentName:"p"},"SELECT")," statement based on column name. This\ndiffers from SQL standard behavior, where columns are inserted based on position.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},(0,i.yg)("inlineCode",{parentName:"p"},"INSERT")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"REPLACE")," do not support all options available in ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/ingestion/ingestion-spec"},"ingestion specs"),",\nincluding the ",(0,i.yg)("inlineCode",{parentName:"p"},"createBitmapIndex")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"multiValueHandling")," ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/ingestion/ingestion-spec#dimension-objects"},"dimension"),"\nproperties, and the ",(0,i.yg)("inlineCode",{parentName:"p"},"indexSpec")," ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/ingestion/ingestion-spec#tuningconfig"},(0,i.yg)("inlineCode",{parentName:"a"},"tuningConfig"))," property."))),(0,i.yg)("h2",{id:"extern-function"},(0,i.yg)("inlineCode",{parentName:"h2"},"EXTERN")," Function"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"The ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/ingestion/ingestion-spec#inclusions-and-exclusions"},"schemaless dimensions"),"\nfeature is not available. All columns and their types must be specified explicitly using the ",(0,i.yg)("inlineCode",{parentName:"p"},"signature")," parameter\nof the ",(0,i.yg)("a",{parentName:"p",href:"/docs/29.0.0/multi-stage-query/reference#extern-function"},(0,i.yg)("inlineCode",{parentName:"a"},"EXTERN")," function"),".")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},(0,i.yg)("inlineCode",{parentName:"p"},"EXTERN")," with input sources that match large numbers of files may exhaust available memory on the controller task.")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},(0,i.yg)("inlineCode",{parentName:"p"},"EXTERN")," refers to external files. Use ",(0,i.yg)("inlineCode",{parentName:"p"},"FROM")," to access ",(0,i.yg)("inlineCode",{parentName:"p"},"druid")," input sources."))))}g.isMDXComponent=!0}}]); |