| "use strict";(self.webpackChunkrocketmq_docs=self.webpackChunkrocketmq_docs||[]).push([[5732],{15680:(e,n,c)=>{c.d(n,{xA:()=>p,yg:()=>y});var t=c(96540);function r(e,n,c){return n in e?Object.defineProperty(e,n,{value:c,enumerable:!0,configurable:!0,writable:!0}):e[n]=c,e}function o(e,n){var c=Object.keys(e);if(Object.getOwnPropertySymbols){var t=Object.getOwnPropertySymbols(e);n&&(t=t.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),c.push.apply(c,t)}return c}function a(e){for(var n=1;n<arguments.length;n++){var c=null!=arguments[n]?arguments[n]:{};n%2?o(Object(c),!0).forEach((function(n){r(e,n,c[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(c)):o(Object(c)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(c,n))}))}return e}function l(e,n){if(null==e)return{};var c,t,r=function(e,n){if(null==e)return{};var c,t,r={},o=Object.keys(e);for(t=0;t<o.length;t++)c=o[t],n.indexOf(c)>=0||(r[c]=e[c]);return r}(e,n);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(t=0;t<o.length;t++)c=o[t],n.indexOf(c)>=0||Object.prototype.propertyIsEnumerable.call(e,c)&&(r[c]=e[c])}return r}var s=t.createContext({}),i=function(e){var n=t.useContext(s),c=n;return e&&(c="function"==typeof e?e(n):a(a({},n),e)),c},p=function(e){var n=i(e.components);return t.createElement(s.Provider,{value:n},e.children)},u="mdxType",g={inlineCode:"code",wrapper:function(e){var n=e.children;return t.createElement(t.Fragment,{},n)}},d=t.forwardRef((function(e,n){var c=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=i(c),d=r,y=u["".concat(s,".").concat(d)]||u[d]||g[d]||o;return c?t.createElement(y,a(a({ref:n},p),{},{components:c})):t.createElement(y,a({ref:n},p))}));function y(e,n){var c=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var o=c.length,a=new Array(o);a[0]=d;var l={};for(var s in n)hasOwnProperty.call(n,s)&&(l[s]=n[s]);l.originalType=e,l[u]="string"==typeof e?e:r,a[1]=l;for(var i=2;i<o;i++)a[i]=c[i];return t.createElement.apply(null,a)}return t.createElement.apply(null,c)}d.displayName="MDXCreateElement"},45223:(e,n,c)=>{c.r(n),c.d(n,{assets:()=>s,contentTitle:()=>a,default:()=>g,frontMatter:()=>o,metadata:()=>l,toc:()=>i});var t=c(58168),r=(c(96540),c(15680));const o={},a="RocketMQ Connect\u5b9e\u62185",l={unversionedId:"connect/08RocketMQ Connect In Action5-ES",id:"version-5.0/connect/08RocketMQ Connect In Action5-ES",title:"RocketMQ Connect\u5b9e\u62185",description:"Elasticsearch Source -> RocketMQ Connect -> Elasticsearch Sink",source:"@site/versioned_docs/version-5.0/10-connect/08RocketMQ Connect In Action5-ES.md",sourceDirName:"10-connect",slug:"/connect/08RocketMQ Connect In Action5-ES",permalink:"/zh/docs/connect/08RocketMQ Connect In Action5-ES",draft:!1,editUrl:"https://github.com/apache/rocketmq-site/tree/new-official-website/versioned_docs/version-5.0/10-connect/08RocketMQ Connect In Action5-ES.md",tags:[],version:"5.0",frontMatter:{},sidebar:"version-5.0/myAutogeneratedSidebar",previous:{title:"RocketMQ Connect\u5b9e\u62184",permalink:"/zh/docs/connect/07RocketMQ Connect In Action4"},next:{title:"RocketMQ Streams \u6982\u89c8",permalink:"/zh/docs/streams/01RocketMQ Streams Overview"}},s={},i=[{value:"\u51c6\u5907",id:"\u51c6\u5907",level:2},{value:"\u542f\u52a8RocketMQ",id:"\u542f\u52a8rocketmq",level:3},{value:"\u6784\u5efa Connector Runtime",id:"\u6784\u5efa-connector-runtime",level:3},{value:"\u6784\u5efa Elasticsearch Connector Plugin",id:"\u6784\u5efa-elasticsearch-connector-plugin",level:3},{value:"\u5355\u673a\u6a21\u5f0f\u8fd0\u884c Connector Worker",id:"\u5355\u673a\u6a21\u5f0f\u8fd0\u884c-connector-worker",level:3},{value:"\u642d\u5efa Elasticsearch \u670d\u52a1",id:"\u642d\u5efa-elasticsearch-\u670d\u52a1",level:3},{value:"\u642d\u5efa Kibana \u670d\u52a1",id:"\u642d\u5efa-kibana-\u670d\u52a1",level:3},{value:"\u5411\u6e90\u7aefES\u5199\u5165\u6d4b\u8bd5\u6570\u636e",id:"\u5411\u6e90\u7aefes\u5199\u5165\u6d4b\u8bd5\u6570\u636e",level:3},{value:"\u6279\u91cf\u5199\u5165\u6d4b\u8bd5\u6570\u636e",id:"\u6279\u91cf\u5199\u5165\u6d4b\u8bd5\u6570\u636e",level:4},{value:"\u67e5\u6570\u636e",id:"\u67e5\u6570\u636e",level:4},{value:"\u5220\u9664\u6570\u636e",id:"\u5220\u9664\u6570\u636e",level:4},{value:"\u542f\u52a8Connector",id:"\u542f\u52a8connector",level:2},{value:"\u542f\u52a8Elasticsearch source connector",id:"\u542f\u52a8elasticsearch-source-connector",level:3},{value:"\u542f\u52a8 Elasticsearch sink connector",id:"\u542f\u52a8-elasticsearch-sink-connector",level:3}],p={toc:i},u="wrapper";function g(e){let{components:n,...c}=e;return(0,r.yg)(u,(0,t.A)({},p,c,{components:n,mdxType:"MDXLayout"}),(0,r.yg)("h1",{id:"rocketmq-connect\u5b9e\u62185"},"RocketMQ Connect\u5b9e\u62185"),(0,r.yg)("p",null,"Elasticsearch Source -> RocketMQ Connect -> Elasticsearch Sink"),(0,r.yg)("h2",{id:"\u51c6\u5907"},"\u51c6\u5907"),(0,r.yg)("h3",{id:"\u542f\u52a8rocketmq"},"\u542f\u52a8RocketMQ"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Linux/Unix/Mac"),(0,r.yg)("li",{parentName:"ol"},"64bit JDK 1.8+;"),(0,r.yg)("li",{parentName:"ol"},"Maven 3.2.x\u6216\u4ee5\u4e0a\u7248\u672c;"),(0,r.yg)("li",{parentName:"ol"},"\u542f\u52a8 RocketMQ\u3002\u4f7f\u7528",(0,r.yg)("a",{parentName:"li",href:"https://rocketmq.apache.org/docs/4.x/"},"RocketMQ 4.x")," \u6216\n",(0,r.yg)("a",{parentName:"li",href:"https://rocketmq.apache.org/docs/quickStart/01quickstart/"},"RocketMQ 5.x"),"\u7248\u672c\u5747\u53ef;"),(0,r.yg)("li",{parentName:"ol"},"\u5de5\u5177\u6d4b\u8bd5 RocketMQ \u6d88\u606f\u6536\u53d1\u662f\u5426\u6b63\u5e38\u3002\u8be6\u89c1",(0,r.yg)("a",{parentName:"li",href:"https://rocketmq.apache.org/docs/4.x/"},"RocketMQ 4.x")," \u6216\n",(0,r.yg)("a",{parentName:"li",href:"https://rocketmq.apache.org/docs/quickStart/01quickstart/"},"RocketMQ 5.x"),"\u6587\u6863\u3002")),(0,r.yg)("p",null,"\u8fd9\u91cc\u5229\u7528\u73af\u5883\u53d8\u91cfNAMESRV_ADDR\u6765\u544a\u8bc9\u5de5\u5177\u5ba2\u6237\u7aefRocketMQ\u7684NameServer\u5730\u5740\u4e3alocalhost:9876"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"#$ cd distribution/target/rocketmq-4.9.7/rocketmq-4.9.7\n$ cd distribution/target/rocketmq-5.1.4/rocketmq-5.1.4\n\n$ export NAMESRV_ADDR=localhost:9876\n$ sh bin/tools.sh org.apache.rocketmq.example.quickstart.Producer\n SendResult [sendStatus=SEND_OK, msgId= ...\n\n$ sh bin/tools.sh org.apache.rocketmq.example.quickstart.Consumer\n ConsumeMessageThread_%d Receive New Messages: [MessageExt...\n")),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\u8bf4\u660e"),"\uff1aRocketMQ\u5177\u5907\u81ea\u52a8\u521b\u5efaTopic\u548cGroup\u7684\u529f\u80fd\uff0c\u5728\u53d1\u9001\u6d88\u606f\u6216\u8ba2\u9605\u6d88\u606f\u65f6\uff0c\u5982\u679c\u76f8\u5e94\u7684Topic\u6216Group\u4e0d\u5b58\u5728\uff0cRocketMQ\u4f1a\u81ea\u52a8\u521b\u5efa\u5b83\u4eec\u3002\u56e0\u6b64\u4e0d\u9700\u8981\u63d0\u524d\u521b\u5efaTopic\u548cGroup\u3002"),(0,r.yg)("h3",{id:"\u6784\u5efa-connector-runtime"},"\u6784\u5efa Connector Runtime"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"git clone https://github.com/apache/rocketmq-connect.git\n\ncd rocketmq-connect\n\nexport RMQ_CONNECT_HOME=`pwd`\n\nmvn -Prelease-connect -Dmaven.test.skip=true clean install -U\n")),(0,r.yg)("h3",{id:"\u6784\u5efa-elasticsearch-connector-plugin"},"\u6784\u5efa Elasticsearch Connector Plugin"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"cd $RMQ_CONNECT_HOME/connectors/rocketmq-connect-elasticsearch/\n\nmvn clean package -Dmaven.test.skip=true\n")),(0,r.yg)("p",null,"\u5c06 Elasticsearch RocketMQ Connector \u7f16\u8bd1\u597d\u7684\u5305\u653e\u5165Runtime\u52a0\u8f7d\u7684Plugin\u76ee\u5f55"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"mkdir -p /Users/YourUsername/rocketmqconnect/connector-plugins\ncp target/rocketmq-connect-elasticsearch-1.0.0-jar-with-dependencies.jar /Users/YourUsername/rocketmqconnect/connector-plugins\n")),(0,r.yg)("h3",{id:"\u5355\u673a\u6a21\u5f0f\u8fd0\u884c-connector-worker"},"\u5355\u673a\u6a21\u5f0f\u8fd0\u884c Connector Worker"),(0,r.yg)("p",null,(0,r.yg)("inlineCode",{parentName:"p"},"connect-standalone.conf"),"\u4e2d\u914d\u7f6e\u4e86RocketMQ\u8fde\u63a5\u5730\u5740\u7b49\u4fe1\u606f\uff0c\u9700\u8981\u6839\u636e\u4f7f\u7528\u60c5\u51b5\u8fdb\u884c\u4fee\u6539"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"cd $RMQ_CONNECT_HOME/distribution/target/rocketmq-connect-0.0.1-SNAPSHOT/rocketmq-connect-0.0.1-SNAPSHOT\n\nvim conf/connect-standalone.conf\n")),(0,r.yg)("p",null,"\u793a\u4f8b\u914d\u7f6e\u4fe1\u606f\u5982\u4e0b"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'workerId=standalone-worker\nstorePathRootDir=/Users/YourUsername/rocketmqconnect/storeRoot\n\n## Http port for user to access REST API\nhttpPort=8082\n\n# Rocketmq namesrvAddr\nnamesrvAddr=localhost:9876\n\n# RocketMQ acl\naclEnable=false\n#accessKey=rocketmq\n#secretKey=12345678\n\nclusterName="DefaultCluster"\n\n# \u63d2\u4ef6\u5730\u5740\uff0c\u7528\u4e8eWorker\u52a0\u8f7dSource/Sink Connector\u63d2\u4ef6\npluginPaths=/Users/YourUsername/rocketmqconnect/connector-plugins\n')),(0,r.yg)("p",null,"\u5355\u673a\u6a21\u5f0f\uff08standalone\uff09\u4e0b\uff0cRocketMQ Connect \u4f1a\u628a\u540c\u6b65\u4f4d\u70b9\u4fe1\u606f\u6301\u4e45\u5316\u5230\u672c\u5730\u6587\u4ef6\u76ee\u5f55 storePathRootDir"),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"storePathRootDir=/Users/YourUsername/rocketmqconnect/storeRoot")),(0,r.yg)("p",null,"\u5982\u679c\u60f3\u91cd\u7f6e\u540c\u6b65\u4f4d\u70b9\uff0c\u5219\u9700\u8981\u5220\u9664\u6301\u4e45\u5316\u7684\u4f4d\u70b9\u4fe1\u606f\u6587\u4ef6"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"rm -rf /Users/YourUsername/rocketmqconnect/storeRoot/*\n")),(0,r.yg)("p",null,"\u91c7\u7528\u5355\u673a\u6a21\u5f0f\u542f\u52a8Connector Worker"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"sh bin/connect-standalone.sh -c conf/connect-standalone.conf &\n")),(0,r.yg)("h3",{id:"\u642d\u5efa-elasticsearch-\u670d\u52a1"},"\u642d\u5efa Elasticsearch \u670d\u52a1"),(0,r.yg)("p",null,"Elasticsearch\u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u5b9e\u65f6\u5206\u5e03\u5f0f\u641c\u7d22\u548c\u5206\u6790\u5f15\u64ce\u3002"),(0,r.yg)("p",null,"\u8fd9\u91cc\u4e3a\u4e86\u65b9\u4fbf\u6f14\u793a\uff0c\u4f7f\u7528 docker \u642d\u5efa 2\u4e2a Elasticsearch \u6570\u636e\u5e93\uff0c\u5206\u522b\u4f5c\u4e3a Connector \u8fde\u63a5\u7684\u6e90\u548c\u76ee\u7684\u7aefES\u6570\u636e\u5e93\u3002"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'docker pull docker.elastic.co/elasticsearch/elasticsearch:7.15.1\n\ndocker run --name es1 -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \\\n -v /Users/YourUsername/rocketmqconnect/es/es1_data:/usr/share/elasticsearch/data \\\n -d docker.elastic.co/elasticsearch/elasticsearch:7.15.1\n\ndocker run --name es2 -p 9201:9200 -p 9301:9300 -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \\\n -v /Users/YourUsername/rocketmqconnect/es/es2_data:/usr/share/elasticsearch/data \\\n -d docker.elastic.co/elasticsearch/elasticsearch:7.15.1\n')),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"docker\u547d\u4ee4\u8bf4\u660e"),"\uff1a"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"--name es2: \u4e3a\u5bb9\u5668\u6307\u5b9a\u4e00\u4e2a\u540d\u79f0\uff0c\u672c\u4f8b\u4e2d\u4e3aes2\u3002"),(0,r.yg)("li",{parentName:"ul"},"-p 9201:9200 -p 9301:9300: \u5c06Elasticsearch\u7684HTTP\u7aef\u53e39200\u548c\u4f20\u8f93\u7aef\u53e39300\u5206\u522b\u6620\u5c04\u5230\u4e3b\u673a\u76849201\u548c9301\u7aef\u53e3\uff0c\u4ee5\u4fbf\u53ef\u4ee5\u901a\u8fc7\u4e3b\u673a\u8bbf\u95eeElasticsearch\u670d\u52a1\u3002"),(0,r.yg)("li",{parentName:"ul"},'-e "discovery.type=single-node": \u8bbe\u7f6eElasticsearch\u7684\u53d1\u73b0\u7c7b\u578b\u4e3a\u5355\u8282\u70b9\u6a21\u5f0f\uff0c\u8fd9\u5bf9\u4e8e\u5355\u673a\u90e8\u7f72\u975e\u5e38\u9002\u7528\u3002'),(0,r.yg)("li",{parentName:"ul"},"-v /Users/YourUsername/rocketmqconnect/es/es2_data:/usr/share/elasticsearch/data: \u5c06\u4e3b\u673a\u4e0a\u7684\u4e00\u4e2a\u76ee\u5f55\u6302\u8f7d\u5230\u5bb9\u5668\u5185\u7684/usr/share/elasticsearch/data\u76ee\u5f55\uff0c\u7528\u4e8e\u6301\u4e45\u5316\u5b58\u50a8Elasticsearch\u6570\u636e\u3002")),(0,r.yg)("p",null,"\u901a\u8fc7\u4ee5\u4e0a\u547d\u4ee4\uff0c\u60a8\u53ef\u4ee5\u8fd0\u884c\u4e00\u4e2a\u5e26\u6709\u81ea\u5b9a\u4e49\u914d\u7f6e\u548c\u6570\u636e\u5b58\u50a8\u7684Elasticsearch\u5bb9\u5668\uff0c\u5e76\u4e14\u53ef\u4ee5\u901a\u8fc7\u4e3b\u673a\u76849200\u7aef\u53e3\u8bbf\u95ee\u5176HTTP API\u3002\u8fd9\u662f\u5728\u672c\u5730\u5f00\u53d1\u6216\u6d4b\u8bd5\u73af\u5883\u4e2d\u8fd0\u884c\u72ec\u7acb\u7684Elasticsearch\u5b9e\u4f8b\u7684\u5e38\u89c1\u65b9\u5f0f\u3002"),(0,r.yg)("p",null,"\u67e5\u770bES\u65e5\u5fd7\uff0c\u67e5\u770b\u542f\u52a8\u662f\u5426\u6709\u62a5\u9519"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"docker logs -f es1\n\ndocker logs -f es2\n")),(0,r.yg)("p",null,"\u4f7f\u7528curl\u547d\u4ee4\u68c0\u67e5Elasticsearch\u662f\u5426\u6b63\u5e38"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"# check es1\ncurl -XGET http://localhost:9200\n\n# check es2\ncurl -XGET http://localhost:9201\n")),(0,r.yg)("p",null,"\u5982\u679c\u6210\u529f\u8fde\u63a5\u5e76\u4e14Elasticsearch\u5df2\u6b63\u5e38\u542f\u52a8\uff0c\u60a8\u5c06\u770b\u5230\u4e0eElasticsearch\u76f8\u5173\u7684\u4fe1\u606f\u548c\u7248\u672c\u53f7\u7684JSON\u54cd\u5e94\u3002"),(0,r.yg)("h3",{id:"\u642d\u5efa-kibana-\u670d\u52a1"},"\u642d\u5efa Kibana \u670d\u52a1"),(0,r.yg)("p",null,"Kibana\u662f\u4e00\u4e2a\u5f00\u6e90\u7684\u6570\u636e\u53ef\u89c6\u5316\u5de5\u5177\uff0c\u7528\u4e8e\u5bf9Elasticsearch\u4e2d\u5b58\u50a8\u7684\u6570\u636e\u8fdb\u884c\u641c\u7d22\u3001\u5206\u6790\u548c\u53ef\u89c6\u5316\u5c55\u793a\u3002\n\u5b83\u63d0\u4f9b\u4e86\u4e30\u5bcc\u7684\u56fe\u8868\u3001\u56fe\u5f62\u548c\u4eea\u8868\u76d8\u7b49\u529f\u80fd\uff0c\u4f7f\u7528\u6237\u80fd\u591f\u4ee5\u76f4\u89c2\u7684\u65b9\u5f0f\u7406\u89e3\u548c\u63a2\u7d22\u6570\u636e\u3002"),(0,r.yg)("p",null,"\u8fd9\u91cc\u4e3a\u4e86\u65b9\u4fbf\u6f14\u793a\uff0c\u4f7f\u7528 docker \u642d\u5efa 2\u4e2a Kibana \u670d\u52a1\uff0c\u5206\u522b\u8fde\u63a5\u524d\u9762\u642d\u5efa\u76842\u4e2aES\u6570\u636e\u5e93\u3002"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"docker pull docker.elastic.co/kibana/kibana:7.15.1\n\ndocker run --name kibana1 --link es1:elasticsearch -p 5601:5601 -d docker.elastic.co/kibana/kibana:7.15.1\n\ndocker run --name kibana2 --link es2:elasticsearch -p 5602:5601 -d docker.elastic.co/kibana/kibana:7.15.1\n\n")),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"docker\u547d\u4ee4\u8bf4\u660e"),"\uff1a"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"--name kibana2: \u4e3a\u5bb9\u5668\u6307\u5b9a\u4e00\u4e2a\u540d\u79f0\uff0c\u672c\u4f8b\u4e2d\u4e3akibana2\u3002"),(0,r.yg)("li",{parentName:"ul"},"--link es2:elasticsearch: \u5c06\u5bb9\u5668\u94fe\u63a5\u5230\u53e6\u4e00\u4e2a\u540d\u4e3aes2\u7684Elasticsearch\u5bb9\u5668\u3002\u8fd9\u5c06\u5141\u8bb8Kibana\u5b9e\u4f8b\u8fde\u63a5\u548c\u4e0eElasticsearch\u8fdb\u884c\u901a\u4fe1\u3002"),(0,r.yg)("li",{parentName:"ul"},"-p 5602:5601: \u5c06Kibana\u7684\u9ed8\u8ba4\u7aef\u53e35601\u6620\u5c04\u5230\u4e3b\u673a\u76845602\u7aef\u53e3\uff0c\u4ee5\u4fbf\u53ef\u4ee5\u901a\u8fc7\u4e3b\u673a\u8bbf\u95eeKibana\u7684\u7528\u6237\u754c\u9762\u3002"),(0,r.yg)("li",{parentName:"ul"},"-d: \u5728\u540e\u53f0\u8fd0\u884c\u5bb9\u5668\u3002")),(0,r.yg)("p",null,"\u901a\u8fc7\u4ee5\u4e0a\u547d\u4ee4\uff0c\u60a8\u53ef\u4ee5\u5728Docker\u5bb9\u5668\u4e2d\u542f\u52a8\u4e00\u4e2a\u72ec\u7acb\u7684Kibana\u5b9e\u4f8b\uff0c\u5e76\u5c06\u5176\u8fde\u63a5\u5230\u53e6\u4e00\u4e2a\u6b63\u5728\u8fd0\u884c\u7684Elasticsearch\u5b9e\u4f8b\u3002\n\u8fd9\u6837\uff0c\u60a8\u53ef\u4ee5\u901a\u8fc7\u6d4f\u89c8\u5668\u8bbf\u95ee\u4e3b\u673a\u76845601\u30015602\u7aef\u53e3\uff0c\u6765\u5206\u522b\u8bbf\u95eeKibana1\u3001Kibana2\u63a7\u5236\u53f0\u3002"),(0,r.yg)("p",null,"\u67e5\u770bKibana\u65e5\u5fd7\uff0c\u67e5\u770b\u542f\u52a8\u662f\u5426\u6709\u62a5\u9519"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"docker logs -f kibana1\n\ndocker logs -f kibana2\n")),(0,r.yg)("p",null,"\u4f7f\u7528\u6d4f\u89c8\u5668\u8bbf\u95ee kibana \u63a7\u5236\u53f0\uff0c\u5730\u5740"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"kibana1: http://localhost:5601"),(0,r.yg)("li",{parentName:"ul"},"kibana2\uff1ahttp://localhost:5602")),(0,r.yg)("p",null,"\u5982\u679c\u63a7\u5236\u53f0\u9875\u9762\u80fd\u6b63\u5e38\u6253\u5f00\uff0c\u5219\u8bf4\u660eKibana\u5df2\u6b63\u5e38\u542f\u52a8\u3002"),(0,r.yg)("h3",{id:"\u5411\u6e90\u7aefes\u5199\u5165\u6d4b\u8bd5\u6570\u636e"},"\u5411\u6e90\u7aefES\u5199\u5165\u6d4b\u8bd5\u6570\u636e"),(0,r.yg)("p",null,"Kibana \u7684 Dev Tools \u53ef\u4ee5\u5e2e\u52a9\u60a8\u5728 Kibana \u4e2d\u4e0e Elasticsearch \u8fdb\u884c\u76f4\u63a5\u7684\u4ea4\u4e92\u548c\u64cd\u4f5c\uff0c\u6267\u884c\u5404\u79cd\u67e5\u8be2\u548c\u64cd\u4f5c\uff0c\u5e76\u5206\u6790\u548c\u7406\u89e3\u8fd4\u56de\u7684\u6570\u636e\u3002\n\u53c2\u89c1\u6587\u6863 ",(0,r.yg)("a",{parentName:"p",href:"https://www.elastic.co/guide/en/kibana/8.9/console-kibana.html"},"console-kibana"),"\u3002"),(0,r.yg)("h4",{id:"\u6279\u91cf\u5199\u5165\u6d4b\u8bd5\u6570\u636e"},"\u6279\u91cf\u5199\u5165\u6d4b\u8bd5\u6570\u636e"),(0,r.yg)("p",null,"\u6d4f\u89c8\u5668\u8bbf\u95eeKibana1\u63a7\u5236\u53f0\uff0c\u5de6\u4fa7\u83dc\u5355\u627e\u5230Dev Tools\uff0c\u8fdb\u5165\u9875\u9762\u540e\u8f93\u5165\u5982\u4e0b\u547d\u4ee4\u5199\u5165\u6d4b\u8bd5\u6570\u636e"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'POST /_bulk\n{ "index" : { "_index" : "connect_es" } }\n{ "id": "1", "field1": "value1", "field2": "value2" }\n{ "index" : { "_index" : "connect_es" } }\n{ "id": "2", "field1": "value3", "field2": "value4" }\n')),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\u8bf4\u660e"),"\uff1a"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"connect_es\uff1a\u6570\u636e\u7684\u7d22\u5f15\u540d\u79f0"),(0,r.yg)("li",{parentName:"ul"},"id/field1/field2\uff1a\u6570\u636e\u4e2d\u7684\u5b57\u6bb5\u540d\u79f0\uff0c1\u3001value1\u3001value2 \u5206\u522b\u662f\u5b57\u6bb5\u7684\u503c\u3002")),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\u6ce8\u610f"),"\uff1a",(0,r.yg)("inlineCode",{parentName:"p"},"rocketmq-connect-elasticsearch")," \u5b58\u5728\u4e00\u4e2a\u9650\u5236\uff0c\u5c31\u662f\u6570\u636e\u4e2d\u5fc5\u987b\u8981\u4e00\u4e2a\u53ef\u7528\u4e8e >= \u6bd4\u8f83\u8fd0\u7b97\u7684\u5b57\u6bb5\uff08\u5b57\u7b26\u4e32 \u6216 \u6570\u5b57\uff09\uff0c\u8be5\u5b57\u6bb5\u4f1a\u88ab\u7528\u4e8e\u8bb0\u5f55\u540c\u6b65\u7684\u4f4d\u70b9\u4fe1\u606f\u3002\n\u4e0a\u9762\u7684\u793a\u4f8b\u4e2d ",(0,r.yg)("inlineCode",{parentName:"p"},"id")," \u5b57\u6bb5\uff0c\u5c31\u662f\u4e00\u4e2a\u5168\u5c40\u552f\u4e00\u3001\u81ea\u589e\u7684\u6570\u503c\u7c7b\u578b\u5b57\u6bb5\u3002"),(0,r.yg)("h4",{id:"\u67e5\u6570\u636e"},"\u67e5\u6570\u636e"),(0,r.yg)("p",null,"\u67e5\u8be2\u7d22\u5f15\u4e0b\u7684\u6570\u636e\uff1a"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'GET /connect_es/_search\n{\n "size": 100\n}\n')),(0,r.yg)("p",null,"\u82e5\u65e0\u6570\u636e\uff0c\u5219\u8fd4\u56de\u793a\u4f8b\u4e3a\uff1a"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'{\n "error" : {\n ... \n "type" : "index_not_found_exception",\n "reason" : "no such index [connect_es]",\n "resource.type" : "index_or_alias",\n "resource.id" : "connect_es",\n "index_uuid" : "_na_",\n "index" : "connect_es"\n },\n "status" : 404\n}\n')),(0,r.yg)("p",null,"\u82e5\u6709\u6570\u636e\uff0c\u5219\u8fd4\u56de\u793a\u4f8b\u4e3a\uff1a"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'{\n ...\n "hits" : {\n "total" : {\n "value" : 2,\n "relation" : "eq"\n },\n "max_score" : 1.0,\n "hits" : [\n {\n "_index" : "connect_es",\n "_type" : "_doc",\n "_id" : "_dx49osBb46Z9cN4hYCg",\n "_score" : 1.0,\n "_source" : {\n "id" : "1",\n "field1" : "value1",\n "field2" : "value2"\n }\n },\n {\n "_index" : "connect_es",\n "_type" : "_doc",\n "_id" : "_tx49osBb46Z9cN4hYCg",\n "_score" : 1.0,\n "_source" : {\n "id" : "2",\n "field1" : "value3",\n "field2" : "value4"\n }\n }\n ]\n }\n}\n\n')),(0,r.yg)("h4",{id:"\u5220\u9664\u6570\u636e"},"\u5220\u9664\u6570\u636e"),(0,r.yg)("p",null,"\u5982\u679c\u56e0\u91cd\u590d\u6d4b\u8bd5\u7b49\u539f\u56e0\uff0c\u9700\u8981\u5220\u9664\u7d22\u5f15\u4e0b\u7684\u6570\u636e\uff0c\u5219\u53ef\u4f7f\u7528\u5982\u4e0b\u547d\u4ee4"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"DELETE /connect_es\n")),(0,r.yg)("h2",{id:"\u542f\u52a8connector"},"\u542f\u52a8Connector"),(0,r.yg)("h3",{id:"\u542f\u52a8elasticsearch-source-connector"},"\u542f\u52a8Elasticsearch source connector"),(0,r.yg)("p",null,"\u8fd0\u884c\u4ee5\u4e0b\u547d\u4ee4\u542f\u52a8 ES source connector\uff0cconnector\u5c06\u4f1a\u8fde\u63a5\u5230ES\u8bfb\u53d6 connect_es \u7d22\u5f15\u4e0b\u7684\u6587\u6863\u6570\u636e\uff0c\n\u5e76\u89e3\u6790 Elasticsearch \u6587\u6863\u6570\u636e\u5c01\u88c5\u6210\u901a\u7528\u7684ConnectRecord\u5bf9\u8c61\uff0c\u53d1\u9001\u5230RocketMQ Topic\u5f53\u4e2d\uff0c \u4f9bSink Connector\u8fdb\u884c\u6d88\u8d39\u3002"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'curl -X POST -H "Content-Type: application/json" http://127.0.0.1:8082/connectors/elasticsearchSourceConnector -d \'{\n "connector.class":"org.apache.rocketmq.connect.elasticsearch.connector.ElasticsearchSourceConnector",\n "elasticsearchHost":"localhost",\n "elasticsearchPort":9200,\n "index":{\n "connect_es": {\n "primaryShards":1,\n "id":1\n }\n },\n "max.tasks":2,\n "connect.topicname":"ConnectEsTopic",\n "value.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter",\n "key.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter"\n}\'\n')),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\u8bf4\u660e"),"\uff1a\u542f\u52a8\u547d\u4ee4\u4e2d\u6307\u5b9a\u4e86\u6e90\u7aefES\u8981\u540c\u6b65\u7684\u7d22\u5f15\u4e3a connect_es \uff0c\u4ee5\u53ca \u7d22\u5f15\u4e0b\u81ea\u589e\u7684\u5b57\u6bb5\u4e3a id \uff0c\u5e76\u4eceid=1\u5f00\u59cb\u62c9\u53d6\u6570\u636e\u3002"),(0,r.yg)("p",null,"curl\u8bf7\u6c42\u8fd4\u56destatus:200\u5219\u8868\u793a\u521b\u5efa\u6210\u529f\uff0c\u8fd4\u56de\u6837\u4f8b\uff1a"),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},'{"status":200,"body":{"connector.class":"...')),(0,r.yg)("p",null,"\u770b\u5230\u4ee5\u4e0b\u65e5\u5fd7\u8bf4\u660e file source connector \u542f\u52a8\u6210\u529f\u4e86"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"tail -100f ~/logs/rocketmqconnect/connect_runtime.log\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Start connector elasticsearchSourceConnector and set target state STARTED successed!!")),(0,r.yg)("h3",{id:"\u542f\u52a8-elasticsearch-sink-connector"},"\u542f\u52a8 Elasticsearch sink connector"),(0,r.yg)("p",null,"\u8fd0\u884c\u4ee5\u4e0b\u547d\u4ee4\u542f\u52a8 ES sink connector\uff0cconnector\u5c06\u4f1a\u8ba2\u9605RocketMQ Topic\u7684\u6570\u636e\u8fdb\u884c\u6d88\u8d39\uff0c\n\u5e76\u5c06\u6bcf\u4e2a\u6d88\u606f\u8f6c\u6362\u4e3a\u6587\u6863\u6570\u636e\u5199\u5165\u5230\u76ee\u7684\u7aefES\u5f53\u4e2d\u3002"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'curl -X POST -H "Content-Type: application/json" http://127.0.0.1:8082/connectors/elasticsearchSinkConnector -d \'{\n "connector.class":"org.apache.rocketmq.connect.elasticsearch.connector.ElasticsearchSinkConnector",\n "elasticsearchHost":"localhost",\n "elasticsearchPort":9201,\n "max.tasks":2,\n "connect.topicnames":"ConnectEsTopic",\n "value.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter",\n "key.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter"\n}\'\n')),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\u8bf4\u660e"),"\uff1a\u542f\u52a8\u547d\u4ee4\u4e2d\u6307\u5b9a\u4e86\u76ee\u7684\u7aefES\u5730\u5740\u548c\u7aef\u53e3\uff0c\u5bf9\u5e94\u4e4b\u524ddocker\u542f\u52a8\u7684es2\u3002"),(0,r.yg)("p",null,"curl\u8bf7\u6c42\u8fd4\u56destatus:200\u5219\u8868\u793a\u521b\u5efa\u6210\u529f\uff0c\u8fd4\u56de\u6837\u4f8b\uff1a"),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},'{"status":200,"body":{"connector.class":"...')),(0,r.yg)("p",null,"\u770b\u5230\u4ee5\u4e0b\u65e5\u5fd7\u8bf4\u660e file source connector \u542f\u52a8\u6210\u529f\u4e86"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"tail -100f ~/logs/rocketmqconnect/connect_runtime.log\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Start connector elasticsearchSinkConnector and set target state STARTED successed!!")),(0,r.yg)("p",null,"\u67e5\u770bsink connector\u662f\u5426\u5c06\u6570\u636e\u5199\u5165\u4e86\u76ee\u7684\u7aefES\u7684\u7d22\u5f15\u5f53\u4e2d\uff1a"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"\u6d4f\u89c8\u5668\u8bbf\u95ee Kibana2 \u63a7\u5236\u53f0\u5730\u5740 http://localhost:5602"),(0,r.yg)("li",{parentName:"ol"},"Kibana2 Dev Tools \u9875\u9762\uff0c\u67e5\u8be2\u7d22\u5f15\u4e0b\u7684\u6570\u636e\uff0c\u82e5\u8ddf\u6e90\u7aef es1 \u4e2d\u7684\u6570\u636e\u4e00\u81f4\u5219\u8bf4\u660eConnector\u8fd0\u884c\u6b63\u5e38\u3002")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'GET /connect_es/_search\n{\n "size": 100\n}\n')))}g.isMDXComponent=!0}}]); |