| "use strict";(self.webpackChunkdoris_website=self.webpackChunkdoris_website||[]).push([[84209],{15680:(e,a,t)=>{t.d(a,{xA:()=>c,yg:()=>m});var n=t(296540);function r(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function l(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);a&&(n=n.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,n)}return t}function i(e){for(var a=1;a<arguments.length;a++){var t=null!=arguments[a]?arguments[a]:{};a%2?l(Object(t),!0).forEach((function(a){r(e,a,t[a])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):l(Object(t)).forEach((function(a){Object.defineProperty(e,a,Object.getOwnPropertyDescriptor(t,a))}))}return e}function o(e,a){if(null==e)return{};var t,n,r=function(e,a){if(null==e)return{};var t,n,r={},l=Object.keys(e);for(n=0;n<l.length;n++)t=l[n],a.indexOf(t)>=0||(r[t]=e[t]);return r}(e,a);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n<l.length;n++)t=l[n],a.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var s=n.createContext({}),u=function(e){var a=n.useContext(s),t=a;return e&&(t="function"==typeof e?e(a):i(i({},a),e)),t},c=function(e){var a=u(e.components);return n.createElement(s.Provider,{value:a},e.children)},d="mdxType",g={inlineCode:"code",wrapper:function(e){var a=e.children;return n.createElement(n.Fragment,{},a)}},p=n.forwardRef((function(e,a){var t=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,c=o(e,["components","mdxType","originalType","parentName"]),d=u(t),p=r,m=d["".concat(s,".").concat(p)]||d[p]||g[p]||l;return t?n.createElement(m,i(i({ref:a},c),{},{components:t})):n.createElement(m,i({ref:a},c))}));function m(e,a){var t=arguments,r=a&&a.mdxType;if("string"==typeof e||r){var l=t.length,i=new Array(l);i[0]=p;var o={};for(var s in a)hasOwnProperty.call(a,s)&&(o[s]=a[s]);o.originalType=e,o[d]="string"==typeof e?e:r,i[1]=o;for(var u=2;u<l;u++)i[u]=t[u];return n.createElement.apply(null,i)}return n.createElement.apply(null,t)}p.displayName="MDXCreateElement"},160565:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>s,contentTitle:()=>i,default:()=>g,frontMatter:()=>l,metadata:()=>o,toc:()=>u});var n=t(58168),r=(t(296540),t(15680));const l={title:"CloudCanal Data Import",language:"en"},i=void 0,o={unversionedId:"ecosystem/cloudcanal",id:"version-2.0/ecosystem/cloudcanal",title:"CloudCanal Data Import",description:"\x3c!--",source:"@site/versioned_docs/version-2.0/ecosystem/cloudcanal.md",sourceDirName:"ecosystem",slug:"/ecosystem/cloudcanal",permalink:"/docs/2.0/ecosystem/cloudcanal",draft:!1,tags:[],version:"2.0",frontMatter:{title:"CloudCanal Data Import",language:"en"},sidebar:"docs",previous:{title:"Beats Doris Output Plugin",permalink:"/docs/2.0/ecosystem/beats"},next:{title:"AutoMQ Load",permalink:"/docs/2.0/ecosystem/automq-load"}},s={},u=[{value:"Introduction",id:"introduction",level:2},{value:"Download",id:"download",level:2},{value:"Function Description",id:"function-description",level:2},{value:"Instructions for Use",id:"instructions-for-use",level:2},{value:"Prerequisites",id:"prerequisites",level:3},{value:"Add DataSource",id:"add-datasource",level:3},{value:"Create DataJob",id:"create-datajob",level:3}],c={toc:u},d="wrapper";function g(e){let{components:a,...l}=e;return(0,r.yg)(d,(0,n.A)({},c,l,{components:a,mdxType:"MDXLayout"}),(0,r.yg)("h1",{id:"cloudcanal-data-import"},"CloudCanal Data Import"),(0,r.yg)("h2",{id:"introduction"},"Introduction"),(0,r.yg)("p",null,"CloudCanal Community Edition is a free data migration and synchronization platform published by ",(0,r.yg)("a",{parentName:"p",href:"https://www.cloudcanalx.com"},"ClouGence Co., Ltd")," that integrates Schema Migration, Full Data Migration, verification, Correction, and real-time Incremental Synchronization.\nCloudCanal help users build a modern data stack in a simple way.\n",(0,r.yg)("img",{alt:"image.png",src:t(229983).A,width:"2360",height:"1032"})),(0,r.yg)("h2",{id:"download"},"Download"),(0,r.yg)("p",null,(0,r.yg)("a",{parentName:"p",href:"https://www.cloudcanalx.com"},"CloudCanal Download Link")),(0,r.yg)("h2",{id:"function-description"},"Function Description"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"It is highly recommended to utilize CloudCanal version 2.2.5.0 or higher for efficient data import into Doris.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"It is advisable to exercise control over the ingestion frequency when using CloudCanal to import ",(0,r.yg)("strong",{parentName:"p"},"incremental data")," into Doris. The default import frequency for writing data from CloudCanal to Doris can be adjusted using the ",(0,r.yg)("inlineCode",{parentName:"p"},"realFlushPauseSec")," parameter, which is set to 10 seconds by default.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"In the current community edition with a maximum memory configuration of 2GB, if DataJobs encounter OOM exceptions or significant GC pauses, it is recommended to reduce the batch size to minimize memory usage. For full DataTask, you can adjust the ",(0,r.yg)("inlineCode",{parentName:"p"},"fullBatchSize")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"fullRingBufferSize")," parameters. For incremental DataTask, the ",(0,r.yg)("inlineCode",{parentName:"p"},"increBatchSize")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"increRingBufferSize")," parameters can be adjusted accordingly.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Supported Source endpoints and features\uff1a"),(0,r.yg)("table",{parentName:"li"},(0,r.yg)("thead",{parentName:"table"},(0,r.yg)("tr",{parentName:"thead"},(0,r.yg)("th",{parentName:"tr",align:null},"Source Endpoints \\ Feature"),(0,r.yg)("th",{parentName:"tr",align:null},"Schema Migration"),(0,r.yg)("th",{parentName:"tr",align:null},"Full Data"),(0,r.yg)("th",{parentName:"tr",align:null},"Incremental"),(0,r.yg)("th",{parentName:"tr",align:null},"Verification"))),(0,r.yg)("tbody",{parentName:"table"},(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},"Oracle"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},"PostgreSQL"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},"Greenplum"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"No"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},"MySQL"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes"),(0,r.yg)("td",{parentName:"tr",align:null},"Yes")))))),(0,r.yg)("h2",{id:"instructions-for-use"},"Instructions for Use"),(0,r.yg)("p",null,"CloudCanal offers a comprehensive productized capability where users can seamlessly add DataSources and create DataJobs through a visual interface. This enables automated schema migration, full data migration, and real-time incremental synchronization. The following example demonstrates how to migrate and synchronize data from a MySQL to the target Doris. Similar procedures can be applied when synchronizing other source endpoints with Doris."),(0,r.yg)("h3",{id:"prerequisites"},"Prerequisites"),(0,r.yg)("p",null,"First, refer to the ",(0,r.yg)("a",{parentName:"p",href:"https://www.cloudcanalx.com/us/cc-doc/quick/quick_start"},"CloudCanal Quick Start")," to complete the installation and deployment of the CloudCanal Community Edition."),(0,r.yg)("h3",{id:"add-datasource"},"Add DataSource"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Log in to the CloudCanal platform"),(0,r.yg)("li",{parentName:"ul"},"Go to ",(0,r.yg)("strong",{parentName:"li"},"DataSource Management")," -> ",(0,r.yg)("strong",{parentName:"li"},"Add DataSource")),(0,r.yg)("li",{parentName:"ul"},"Select ",(0,r.yg)("strong",{parentName:"li"},"Doris")," from the options for self-built databases")),(0,r.yg)("p",null,(0,r.yg)("img",{alt:"image.png",src:t(360822).A,width:"3874",height:"1964"})),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Tips:"),(0,r.yg)("ul",{parentName:"blockquote"},(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Client Address: The address of the Doris server's MySQL client service port. CloudCanal primarily uses this address to query metadata information of the database tables.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"HTTP Address: The HTTP address is mainly used to receive data import requests from CloudCanal.")))),(0,r.yg)("h3",{id:"create-datajob"},"Create DataJob"),(0,r.yg)("p",null,"Once the DataSource has been added successfully, you can follow these steps to create data migration and synchronization DataJob."),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Go to ",(0,r.yg)("strong",{parentName:"li"},"DataJob Management")," -> ",(0,r.yg)("strong",{parentName:"li"},"Create DataJob")," in the CloudCanal"),(0,r.yg)("li",{parentName:"ul"},"Select the source and target databases for the DataJob"),(0,r.yg)("li",{parentName:"ul"},"Click Next Step")),(0,r.yg)("p",null,(0,r.yg)("img",{alt:"image.png",src:t(792589).A,width:"3886",height:"1966"})),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Choose ",(0,r.yg)("strong",{parentName:"li"},"Incremental")," and enable ",(0,r.yg)("strong",{parentName:"li"},"Full Data")),(0,r.yg)("li",{parentName:"ul"},"Select DDL Sync"),(0,r.yg)("li",{parentName:"ul"},"Click Next Step")),(0,r.yg)("p",null,(0,r.yg)("img",{alt:"image.png",src:t(129348).A,width:"3878",height:"1966"})),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Select the tables you want to subscribe to. Please note that ",(0,r.yg)("strong",{parentName:"li"},"the tables automatically created during structural migration follow the primary key model, so tables without a primary key are not currently supported")),(0,r.yg)("li",{parentName:"ul"},"Click Next Step")),(0,r.yg)("p",null,(0,r.yg)("img",{alt:"image.png",src:t(784155).A,width:"3874",height:"1858"})),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Configure the column mapping"),(0,r.yg)("li",{parentName:"ul"},"Click Next Step")),(0,r.yg)("p",null,(0,r.yg)("img",{alt:"image.png",src:t(462610).A,width:"3882",height:"1864"})),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Create DataJob")),(0,r.yg)("p",null,(0,r.yg)("img",{alt:"image.png",src:t(10793).A,width:"3880",height:"1968"})),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Check the status of DataJob. The DataJob will automatically go through the stages of Schema Migration, Full Data, and Incremental after it has been created")),(0,r.yg)("p",null,(0,r.yg)("img",{alt:"image.png",src:t(886592).A,width:"2250",height:"344"})))}g.isMDXComponent=!0},229983:(e,a,t)=>{t.d(a,{A:()=>n});const n=t.p+"assets/images/cloudcanal-1-9111d3c064473b9574930007b65bf7b6.jpg"},360822:(e,a,t)=>{t.d(a,{A:()=>n});const n=t.p+"assets/images/cloudcanal-11-327004bada217f7029c3c12b8d859b67.png"},792589:(e,a,t)=>{t.d(a,{A:()=>n});const n=t.p+"assets/images/cloudcanal-12-d33cdf8cfd3c8f5b99d7828d4f42d867.png"},129348:(e,a,t)=>{t.d(a,{A:()=>n});const n=t.p+"assets/images/cloudcanal-13-fb5c313d13b091f574581da275f0d6ac.png"},784155:(e,a,t)=>{t.d(a,{A:()=>n});const n=t.p+"assets/images/cloudcanal-14-06aaf179e5118c858312bbadc8bd395d.png"},462610:(e,a,t)=>{t.d(a,{A:()=>n});const n=t.p+"assets/images/cloudcanal-15-58468710037c33eb4f931d9e4e373888.png"},10793:(e,a,t)=>{t.d(a,{A:()=>n});const n=t.p+"assets/images/cloudcanal-16-6d529b684e19e85c8966649592aaf3aa.png"},886592:(e,a,t)=>{t.d(a,{A:()=>n});const n=t.p+"assets/images/cloudcanal-17-954bad2113cfeff8d23a89977103a526.png"}}]); |