| <!doctype html> |
| <html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-1.1.3 plugin-docs plugin-id-default docs-doc-id-deployment/start-metadatasource"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="generator" content="Docusaurus v2.2.0"> |
| <title data-rh="true">DataSource | Apache Linkis</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:url" content="https://linkis.apache.org/docs/1.1.3/deployment/start-metadatasource"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="1.1.3"><meta data-rh="true" name="docusaurus_tag" content="docs-default-1.1.3"><meta data-rh="true" name="docsearch:version" content="1.1.3"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-1.1.3"><meta data-rh="true" property="og:title" content="DataSource | Apache Linkis"><meta data-rh="true" name="description" content="Introduce how to use the new feature function data source of version 1.1.0"><meta data-rh="true" property="og:description" content="Introduce how to use the new feature function data source of version 1.1.0"><link data-rh="true" rel="icon" href="/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://linkis.apache.org/docs/1.1.3/deployment/start-metadatasource"><link data-rh="true" rel="alternate" href="https://linkis.apache.org/docs/1.1.3/deployment/start-metadatasource" hreflang="en"><link data-rh="true" rel="alternate" href="https://linkis.apache.org/zh-CN/docs/1.1.3/deployment/start-metadatasource" hreflang="zh-CN"><link data-rh="true" rel="alternate" href="https://linkis.apache.org/docs/1.1.3/deployment/start-metadatasource" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://AE29KQB3IA-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/blog/rss.xml" title="Apache Linkis RSS Feed"> |
| <link rel="alternate" type="application/atom+xml" href="/blog/atom.xml" title="Apache Linkis Atom Feed"> |
| <link rel="alternate" type="application/json" href="/blog/feed.json" title="Apache Linkis JSON Feed"> |
| |
| |
| |
| <link rel="search" type="application/opensearchdescription+xml" title="Apache Linkis" href="/opensearch.xml"><link rel="stylesheet" href="/assets/css/styles.fee819ac.css"> |
| <link rel="preload" href="/assets/js/runtime~main.36b81271.js" as="script"> |
| <link rel="preload" href="/assets/js/main.f09c65a0.js" as="script"> |
| </head> |
| <body class="navigation-with-keyboard"> |
| <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"light")}()</script><div id="__docusaurus"> |
| <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#docusaurus_skipToContent_fallback">Skip to main content</a></div><nav class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo.png" alt="Apache Linkis Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/img/logo.png" alt="Apache Linkis Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">Apache Linkis</b></a><a class="navbar__item navbar__link" href="/download/main">Download</a><a class="navbar__item navbar__link" href="/community/how-to-subscribe">Community</a><a class="navbar__item navbar__link" href="/blog">Blog</a><a class="navbar__item navbar__link" href="/team">Team</a><a class="navbar__item navbar__link" href="/user">Users</a><a href="https://docs.qq.com/doc/DUkdTTGhVSlZ0VXVt" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link">FAQ</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">ASF</a><ul class="dropdown__menu"><li><a href="https://www.apache.org/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Foundation</a></li><li><a href="https://www.apache.org/licenses/" target="_blank" rel="noopener noreferrer" class="dropdown__link">License</a></li><li><a href="https://www.apache.org/events/current-event" target="_blank" rel="noopener noreferrer" class="dropdown__link">Events</a></li><li><a href="https://www.apache.org/security/" target="_blank" rel="noopener noreferrer" class="dropdown__link">Security</a></li><li><a href="https://www.apache.org/foundation/sponsorship.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Sponsorship</a></li><li><a href="https://privacy.apache.org/policies/privacy-policy-public.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Privacy</a></li><li><a href="https://www.apache.org/foundation/thanks.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Thanks</a></li></ul></div></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Doc</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/docs/latest/about/introduction">1.5.0</a></li><li><a class="dropdown__link" href="/docs/1.4.0/about/introduction">1.4.0</a></li><li><a class="dropdown__link" href="/docs/1.3.2/about/introduction">1.3.2</a></li><li><a class="dropdown__link" href="/docs/1.6.0/about/introduction">Next(1.6.0)</a></li><li><a class="dropdown__link" href="/versions">All Version</a></li></ul></div><a href="https://github.com/apache/linkis" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub"></a><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/docs/1.1.3/deployment/start-metadatasource" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li><li><a href="/zh-CN/docs/1.1.3/deployment/start-metadatasource" target="_self" rel="noopener noreferrer" class="dropdown__link" lang="zh-CN">简体中文</a></li></ul></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebar_njMd"><nav class="menu thin-scrollbar menu_SIkG"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.1.3/introduction">Introduction</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/docs/1.1.3/release">Version overview</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/docs/1.1.3/deployment/quick-deploy">Deployment</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/quick-deploy">Quick Deployment</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/cluster-deployment">Cluster Deployment</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/engine-conn-plugin-installation">EngineConnPlugin Installation</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/installation-hierarchical-structure">Installation Directory Structure</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/unpack-hierarchical-structure">installation package directory structure</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/sourcecode-hierarchical-structure">Source Code Directory Structure</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/web-install">Linkis Console Deployment</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/involve-skywalking-into-linkis">Involve SkyWaling into Linkis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/docs/1.1.3/deployment/start-metadatasource">DataSource</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/deploy-linkis-without-hdfs">Deploy Linkis without HDFS</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/linkis-scriptis-install">Installation and deployment of the tool scriptis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/involve-prometheus-into-linkis">Involve Prometheus into Linkis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/docs/1.1.3/deployment/involve-knife4j-into-linkis">Involve Knife4j into Linkis</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.3/user-guide/overview">User Guide</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.3/engine-usage/overview">Engine Usage</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.3/api/overview">API Docs</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.3/table/udf-table">Table Structure</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.3/architecture/overview">Architecture</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.3/development/linkis-compile-and-package">Development</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.3/upgrade/upgrade-from-0.X-to-1.0-guide">Upgrade Guide</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/docs/1.1.3/tuning-and-troubleshooting/overview">Tuning And Troubleshooting</a></div></li></ul></nav></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->Apache Linkis<!-- --> <b>1.1.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/docs/latest/about/introduction">latest version</a></b> (<!-- -->1.5.0<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_OVgt"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deployment</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">DataSource</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.1.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>DataSource</h1></header><blockquote><p>Introduce how to use the new feature function data source of version 1.1.0</p></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1-data-source-function-introduction">1. Data source function introduction<a class="hash-link" href="#1-data-source-function-introduction" title="Direct link to heading"></a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="11-concept">1.1 Concept<a class="hash-link" href="#11-concept" title="Direct link to heading"></a></h3><ul><li>Data source: We call database services that can provide data storage as databases, such as mysql/hive/kafka. The data source defines the configuration information for connecting to the actual database. The configuration information is mainly the address required for connection and user authentication information , connection parameters, etc. Stored in the linkis<em>ps_dm_datasource</em>* table related to the linkis database</li><li>Metadata: single refers to the metadata of the database, which refers to the data that defines the data structure and the data of various object structures of the database. For example, the database name, table name, column name, field length, type and other information data in the database.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="12-three-main-modules">1.2 Three main modules<a class="hash-link" href="#12-three-main-modules" title="Direct link to heading"></a></h3><p><strong>linkis-datasource-client</strong> |
| Client module, DataSourceRemoteClient for basic management of user data sources, and MetaDataRemoteClient for metadata query operations.</p><p><strong>linkis-datasource-manager-server</strong> |
| Data source management module, service name ps-data-source-manager. Perform basic management of data sources, and provide http interfaces such as adding, querying, modifying, and connection testing of external data sources. The rpc service is provided internally, which is convenient for the data element management module to call through rpc to query the necessary information needed to establish a connection to the database.</p><ul><li><a href="/docs/1.1.3/api/http/linkis-ps-publicservice-api/data-source-manager-api">http interface documentation</a></li><li>http interface class org.apache.linkis.metadatamanager.server.restful</li><li>rpc interface class org.apache.linkis.metadatamanager.server.receiver</li></ul><p><strong>linkis-metedata-manager-server</strong> |
| Data element management module, service name ps-metadatamanager. It provides the basic query function of the data metadata of the database, provides the http interface externally, and provides the rpc service internally, which is convenient for the data source management module to perform the connection test of the data source through the rpc call.</p><ul><li><a href="/docs/1.1.3/api/http/linkis-ps-publicservice-api/metadatamanager-api">http interface documentation</a></li><li>http interface class org.apache.linkis.datasourcemanager.core.restful</li><li>rpc interface class org.apache.linkis.datasourcemanager.core.receivers</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="13-processing-logic">1.3 Processing logic<a class="hash-link" href="#13-processing-logic" title="Direct link to heading"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="131-linkisdatasourceremoteclient">1.3.1 LinkisDataSourceRemoteClient<a class="hash-link" href="#131-linkisdatasourceremoteclient" title="Direct link to heading"></a></h4><p>The functional structure diagram is as follows: |
| <img loading="lazy" alt="datasource" src="/assets/images/datasource-33466f8154b90ce8c308f80af54fed9d.png" width="860" height="661" class="img_ev3q"></p><ul><li>The LinkisDataSourceRemoteClient client assembles the http request according to the request parameters,</li><li>HTTP request sent to linkis-ps-data-source-manager</li><li>linkis-ps-data-source-manager will perform basic parameter verification, some interfaces can only be operated by the administrator role</li><li>linkis-ps-data-source-manager performs basic data operations with the database</li><li>The data source test connection interface provided by linkis-ps-data-source-manager internally uses rpc to call the ps-metadatamanager method for connection test</li><li>The data result after the http request is processed will be mapped and converted from the result set to the entity class by annotating the DWSHttpMessageResult function</li></ul><p>LinkisDataSourceRemoteClient interface</p><ul><li>GetAllDataSourceTypesResult getAllDataSourceTypes(GetAllDataSourceTypesAction) Query all data source types</li><li>QueryDataSourceEnvResult queryDataSourceEnv(QueryDataSourceEnvAction) Query the cluster configuration information that can be used by the data source</li><li>GetInfoByDataSourceIdResult getInfoByDataSourceId(GetInfoByDataSourceIdAction): query data source information by data source id</li><li>QueryDataSourceResult queryDataSource(QueryDataSourceAction) query data source information</li><li>GetConnectParamsByDataSourceIdResult getConnectParams(GetConnectParamsByDataSourceIdAction) Get connection configuration parameters</li><li>CreateDataSourceResult createDataSource(CreateDataSourceAction) to create a data source</li><li>DataSourceTestConnectResult getDataSourceTestConnect(DataSourceTestConnectAction) to test whether the data source can be connected normally</li><li>DeleteDataSourceResult deleteDataSource(DeleteDataSourceAction) deletes the data source</li><li>ExpireDataSourceResult expireDataSource(ExpireDataSourceAction) sets the data source to expired state</li><li>GetDataSourceVersionsResult getDataSourceVersions(GetDataSourceVersionsAction) Query the version list of the data source configuration</li><li>PublishDataSourceVersionResult publishDataSourceVersion(PublishDataSourceVersionAction) publishes the data source configuration version</li><li>UpdateDataSourceResult updateDataSource(UpdateDataSourceAction) to update the data source</li><li>UpdateDataSourceParameterResult updateDataSourceParameter(UpdateDataSourceParameterAction) Update data source configuration parameters</li><li>GetKeyTypeDatasourceResult getKeyDefinitionsByType(GetKeyTypeDatasourceAction) Query the configuration properties required by a data source type</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="132-linkismetadataremoteclient">1.3.2 LinkisMetaDataRemoteClient<a class="hash-link" href="#132-linkismetadataremoteclient" title="Direct link to heading"></a></h4><p>The functional structure diagram is as follows: |
| <img loading="lazy" alt="metadata" src="/assets/images/metadata-00c6c1e49a4eafa8a2496b9a08917758.png" width="931" height="527" class="img_ev3q"></p><ul><li>LinkisMetaDataRemoteClient client, according to the request parameters, assemble the http request,</li><li>HTTP request sent to ps-metadatamanager</li><li>ps-metadatamanager will perform basic parameter verification,</li><li>The request will send an RPC request to linkis-ps-data-source-manager based on the parameter datasourceId to obtain the type of the data source, connection parameters such as username and password, etc.</li><li>After getting the information required for the connection, load the lib package in the corresponding directory according to the data source type, and call the corresponding function method through the reflection mechanism to query the metadata information</li><li>The data result after the http request is processed will be mapped and converted from the result set to the entity class by annotating the DWSHttpMessageResult function</li></ul><p>LinkisMetaDataRemoteClient interface</p><ul><li>MetadataGetDatabasesResult getDatabases(MetadataGetDatabasesAction) query database list</li><li>MetadataGetTablesResult getTables(MetadataGetTablesAction) query table data</li><li>MetadataGetTablePropsResult getTableProps(MetadataGetTablePropsAction)</li><li>MetadataGetPartitionsResult getPartitions(MetadataGetPartitionsAction) query partition table</li><li>MetadataGetColumnsResult getColumns(MetadataGetColumnsAction) query data table fields</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="13-source-module-directory-structure">1.3 Source module directory structure<a class="hash-link" href="#13-source-module-directory-structure" title="Direct link to heading"></a></h3><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">linkis-public-enhancements/linkis-datasource</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">├── linkis-datasource-client //client code</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">├── linkis-datasource-manager //Datasource management module</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── common //Data </span><span class="token builtin class-name" style="color:rgb(189, 147, 249)">source</span><span class="token plain"> management common module</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ └── server //Data </span><span class="token builtin class-name" style="color:rgb(189, 147, 249)">source</span><span class="token plain"> management </span><span class="token function" style="color:rgb(80, 250, 123)">service</span><span class="token plain"> module</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">├── linkis-metadata //Module existing </span><span class="token keyword" style="color:rgb(189, 147, 249);font-style:italic">in</span><span class="token plain"> the old version, reserved</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">├── linkis-metadata-manager //Data Metadata Management Module</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── common //Data element management common module</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── server //Data element management </span><span class="token function" style="color:rgb(80, 250, 123)">service</span><span class="token plain"> module</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ └── </span><span class="token function" style="color:rgb(80, 250, 123)">service</span><span class="token plain"> //Supported data sources</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── elasticsearch</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── hive</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── kafka</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ └── mysql</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="14-installation-package-directory-structure">1.4 Installation package directory structure<a class="hash-link" href="#14-installation-package-directory-structure" title="Direct link to heading"></a></h3><div class="language-shell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-shell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">/lib/linkis-public-enhancements/</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">├── linkis-ps-data-source-manager</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">├── linkis-ps-metadatamanager</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ └── </span><span class="token function" style="color:rgb(80, 250, 123)">service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── elasticsearch</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── hive</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ ├── kafka</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">│ └── mysql</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>wds.linkis.server.mdm.service.lib.dir</code> controls the classpath loaded during reflection calls. The default value of the parameter is <code>/lib/linkis-public-enhancements/linkis-ps-metadatamanager/service</code></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="15-configuration-parameters">1.5 Configuration Parameters<a class="hash-link" href="#15-configuration-parameters" title="Direct link to heading"></a></h3><p>See <a href="/docs/1.1.0/tuning-and-troubleshooting/configuration#6-datasource-and-metadata-service-configuration-parameters">Tuning and Troubleshooting>Parameter List#datasourceConfiguration Parameters</a></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2-enable-data-source-function">2. Enable data source function<a class="hash-link" href="#2-enable-data-source-function" title="Direct link to heading"></a></h2><p>In the startup script of linkis, the two services related to the data source (ps-data-source-manager, ps-metadatamanager) will not be started by default. |
| If you want to use the data source service, you can enable it in the following ways: |
| Modify <code>export ENABLE_METADATA_MANAGER=true</code> in <code>$LINKIS_CONF_DIR/linkis-env.sh</code> to true. |
| When the service is started and stopped through linkis-start-all.sh/linkis-stop-all.sh, the data source service will be started and stopped.</p><p>Check whether the service starts normally through the eureka page</p><p><img loading="lazy" alt="datasource eureka" src="/assets/images/eureka-6c655a155cf632afe0003ad92f7cddd7.png" width="1820" height="450" class="img_ev3q"></p><div class="theme-admonition theme-admonition-caution alert alert--warning admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 16 16"><path fill-rule="evenodd" d="M8.893 1.5c-.183-.31-.52-.5-.887-.5s-.703.19-.886.5L.138 13.499a.98.98 0 0 0 0 1.001c.193.31.53.501.886.501h13.964c.367 0 .704-.19.877-.5a1.03 1.03 0 0 0 .01-1.002L8.893 1.5zm.133 11.497H6.987v-2.003h2.039v2.003zm0-3.004H6.987V5.987h2.039v4.006z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><ul><li><ol><li>Management of linkis The web version needs to be upgraded to version 1.1.0 to use the data source management page function on the linkis console.</li></ol></li><li><ol start="2"><li>At present, there are jar packages of mysql/hive/kafak/elasticsearch in the data source, but the kafak/elasticsearch data source has not been strictly tested, and the complete availability of functions is not guaranteed.</li></ol></li></ul></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-use-of-data-sources">3. Use of data sources<a class="hash-link" href="#3-use-of-data-sources" title="Direct link to heading"></a></h2><p>The use of data sources is divided into three steps:</p><ul><li>step 1. Create data source/configure connection parameters</li><li>step 2. Publish the data source and select the connection configuration version to use</li><li>step 3. Data source usage, query metadata information |
| , hive/kafka/elasticsearch configuration is associated with the corresponding cluster environment configuration.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="31-mysql-data-source">3.1 Mysql data source<a class="hash-link" href="#31-mysql-data-source" title="Direct link to heading"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="311-created-through-the-management-console">3.1.1 Created through the management console<a class="hash-link" href="#311-created-through-the-management-console" title="Direct link to heading"></a></h4><blockquote><p>You can only create configuration data sources, and test whether the data sources can be connected normally, and cannot directly query metadata</p></blockquote><p>Data Source Management > New Data Source > Select MySQL Type</p><p>Enter relevant configuration information</p><p><img loading="lazy" alt="create mysql" src="/assets/images/create_mysql-748ae7a51622be09651e31630bf6403e.png" width="1889" height="751" class="img_ev3q"></p><p>After the entry is successful, you can pass the connection test to verify whether the connection can be made normally</p><div class="theme-admonition theme-admonition-caution alert alert--warning admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 16 16"><path fill-rule="evenodd" d="M8.893 1.5c-.183-.31-.52-.5-.887-.5s-.703.19-.886.5L.138 13.499a.98.98 0 0 0 0 1.001c.193.31.53.501.886.501h13.964c.367 0 .704-.19.877-.5a1.03 1.03 0 0 0 .01-1.002L8.893 1.5zm.133 11.497H6.987v-2.003h2.039v2.003zm0-3.004H6.987V5.987h2.039v4.006z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><ul><li>The system to which the data source created through the management console belongs is Linkis</li><li>After the creation is successful, it needs to be published (switching and selecting the configuration parameter version when publishing) before it can be used normally</li></ul></div></div><p>Publishing of the configuration (using that configuration for the connection to the data source):</p><p>Click on the version and then pop up the page to select the appropriate configuration to publish</p><p><img loading="lazy" alt="publish" src="/assets/images/publish_version-74eb63ec4aedd2b4a28ff93fbb5e567d.png" width="1635" height="574" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="312-using-the-client">3.1.2 Using the client<a class="hash-link" href="#312-using-the-client" title="Direct link to heading"></a></h4><p>scala code example:</p><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">package org.apache.linkis.datasource.client</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import java.util</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import java.util.concurrent.TimeUnit</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.common.utils.JsonUtils</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.datasource.client.request._</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.datasource.client.response._</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.datasourcemanager.common.domain.DataSource</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.junit.jupiter.api.{Disabled, Test}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">object TestMysqlClient {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val gatewayUrl = "http://127.0.0.1:9001"</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val clientConfig = DWSClientConfigBuilder.newBuilder</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .addServerUrl(gatewayUrl)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .connectionTimeout(30000)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .discoveryEnabled(false)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .discoveryFrequency(1, TimeUnit.MINUTES)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .loadbalancerEnabled(true)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .maxConnectionSize(1)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .retryEnabled(false)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .readTimeout(30000)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthenticationStrategy(new StaticAuthenticationStrategy)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthTokenKey("hadoop")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthTokenValue("xxxxx")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDWSVersion("v1")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val dataSourceclient = new LinkisDataSourceRemoteClient(clientConfig.build())</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val clientConfig2 = DWSClientConfigBuilder.newBuilder</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .addServerUrl(gatewayUrl)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .connectionTimeout(30000)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .discoveryEnabled(false)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .discoveryFrequency(1, TimeUnit.MINUTES)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .loadbalancerEnabled(true)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .maxConnectionSize(1)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .retryEnabled(false)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .readTimeout(30000)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthenticationStrategy(new StaticAuthenticationStrategy)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthTokenKey("hadoop")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthTokenValue("xxxxx")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDWSVersion("v1")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig2.build())</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @Test</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @Disabled</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> def testCreateDataSourceMysql: Unit = {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val user = "hadoop"</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val system = "Linkis"</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> //create data source</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val dataSource = new DataSource();</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSource.setDataSourceName("for-mysql-test")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSource.setDataSourceDesc("this is for mysql test")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSource.setCreateSystem(system)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSource.setDataSourceTypeId(1L)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val map = JsonUtils.jackson.readValue(JsonUtils.jackson.writeValueAsString(dataSource), new util.HashMap[String, Any]().getClass)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val createDataSourceAction: CreateDataSourceAction = CreateDataSourceAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .addRequestPayloads(map)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val createDataSourceResult: CreateDataSourceResult = dataSourceclient.createDataSource(createDataSourceAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val dataSourceId = createDataSourceResult.getInsertId</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> // set connection parameters</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val params = new util.HashMap[String, Any]</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val connectParams = new util.HashMap[String, Any]</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> connectParams.put("host", "127.0.0.1")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> connectParams.put("port", "36000")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> connectParams.put("username", "db username")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> connectParams.put("password", "db password")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> params.put("connectParams", connectParams)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> params.put("comment", "init")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val updateParameterAction: UpdateDataSourceParameterAction = UpdateDataSourceParameterAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .addRequestPayloads(params)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val updateParameterResult: UpdateDataSourceParameterResult = dataSourceclient.updateDataSourceParameter(updateParameterAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val version: Long = updateParameterResult.getVersion</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> //publish configuration version</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSourceclient.publishDataSourceVersion(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> PublishDataSourceVersionAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setVersion(version)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build())</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> // use example</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetDatabasesAction: MetadataGetDatabasesAction = MetadataGetDatabasesAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setSystem(system)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetDatabasesResult: MetadataGetDatabasesResult = metaDataClient.getDatabases(metadataGetDatabasesAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetTablesAction: MetadataGetTablesAction = MetadataGetTablesAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDatabase("linkis")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setSystem(system)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetTablesResult: MetadataGetTablesResult = metaDataClient.getTables(metadataGetTablesAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetColumnsAction = MetadataGetColumnsAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDatabase("linkis")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setSystem(system)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setTable("linkis_datasource")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetColumnsResult: MetadataGetColumnsResult = metaDataClient.getColumns(metadataGetColumnsAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> </span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="32-hive-data-source">3.2 Hive data source<a class="hash-link" href="#32-hive-data-source" title="Direct link to heading"></a></h3><h4 class="anchor anchorWithStickyNavbar_LWe7" id="321-created-through-the-management-console">3.2.1 Created through the management console<a class="hash-link" href="#321-created-through-the-management-console" title="Direct link to heading"></a></h4><blockquote><p>You can only create configuration data sources, and test whether the data sources can be connected normally, and cannot directly query metadata</p></blockquote><p>First need to configure the cluster environment information |
| Table <code>linkis_ps_dm_datasource_env</code></p><div class="language-roomsql codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-roomsql codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">INSERT INTO `linkis_ps_dm_datasource_env`</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">(`env_name`, `env_desc`, `datasource_type_id`, `parameter`, `create_user`, `modify_user`)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">VALUES</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">('testEnv', 'TestEnv', 4, '{\r\n "keytab": "4dd408ad-a2f9-4501-83b3-139290977ca2",\r\n "uris": "thrift://clustername:9083 ",\r\n "principle":"hadoop@WEBANK.COM"\r\n}', 'user','user');</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The primary key id, used as the envId, needs to pass the envId parameter to obtain information about the cluster configuration when establishing a connection. |
| Explanation of configuration fields:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">{</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> "keytab": "bml resource id", //keytab stores the resourceId in the material library, which currently needs to be manually uploaded through the http interface.</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> "uris": "thrift://clustername:9083",</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> "principle":"hadoop@WEBANK.COM" //Authenticated principle</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Create on the web side:</p><p><img loading="lazy" alt="create_hive" src="/assets/images/create_hive-6b7da346506ec10f1ebd68ed37ff4e12.png" width="1624" height="554" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="322-using-the-client">3.2.2 Using the client<a class="hash-link" href="#322-using-the-client" title="Direct link to heading"></a></h4><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#F8F8F2;--prism-background-color:#282A36"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#F8F8F2"><span class="token plain">package org.apache.linkis.datasource.client</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import java.util</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import java.util.concurrent.TimeUnit</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.common.utils.JsonUtils</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.datasource.client.request._</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.datasource.client.response._</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.datasourcemanager.common.domain.DataSource</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">import org.junit.jupiter.api.{Disabled, Test}</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">object TestHiveClient {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val gatewayUrl = "http://127.0.0.1:9001"</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val clientConfig = DWSClientConfigBuilder.newBuilder</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .addServerUrl(gatewayUrl)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .connectionTimeout(30000)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .discoveryEnabled(false)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .discoveryFrequency(1, TimeUnit.MINUTES)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .loadbalancerEnabled(true)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .maxConnectionSize(1)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .retryEnabled(false)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .readTimeout(30000)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthenticationStrategy(new StaticAuthenticationStrategy)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthTokenKey("hadoop")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthTokenValue("xxxxx")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDWSVersion("v1")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val dataSourceclient = new LinkisDataSourceRemoteClient(clientConfig.build())</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val clientConfig2 = DWSClientConfigBuilder.newBuilder</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .addServerUrl(gatewayUrl)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .connectionTimeout(30000)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .discoveryEnabled(false)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .discoveryFrequency(1, TimeUnit.MINUTES)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .loadbalancerEnabled(true)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .maxConnectionSize(1)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .retryEnabled(false)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .readTimeout(30000)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthenticationStrategy(new StaticAuthenticationStrategy)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthTokenKey("hadoop")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setAuthTokenValue("xxxxx")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDWSVersion("v1")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig2.build())</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @Test</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> @Disabled</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> def testCreateDataSourceMysql: Unit = {</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val user = "hadoop"</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val system = "Linkis"</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> //create data source</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val dataSource = new DataSource();</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSource.setDataSourceName("for-hive-test")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSource.setDataSourceDesc("this is for hive test")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSource.setCreateSystem(system)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSource.setDataSourceTypeId(4L)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val map = JsonUtils.jackson.readValue(JsonUtils.jackson.writeValueAsString(dataSource), new util.HashMap[String, Any]().getClass)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val createDataSourceAction: CreateDataSourceAction = CreateDataSourceAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .addRequestPayloads(map)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val createDataSourceResult: CreateDataSourceResult = dataSourceclient.createDataSource(createDataSourceAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val dataSourceId = createDataSourceResult.getInsertId</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> // set connection parameters</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val params = new util.HashMap[String, Any]</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val connectParams = new util.HashMap[String, Any]</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> connectParams.put("envId", "3")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> params.put("connectParams", connectParams)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> params.put("comment", "init")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val updateParameterAction: UpdateDataSourceParameterAction = UpdateDataSourceParameterAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .addRequestPayloads(params)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val updateParameterResult: UpdateDataSourceParameterResult = dataSourceclient.updateDataSourceParameter(updateParameterAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val version: Long = updateParameterResult.getVersion</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> //publish configuration version</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> dataSourceclient.publishDataSourceVersion(</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> PublishDataSourceVersionAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setVersion(version)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build())</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> // use example</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetDatabasesAction: MetadataGetDatabasesAction = MetadataGetDatabasesAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setSystem(system)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetDatabasesResult: MetadataGetDatabasesResult = metaDataClient.getDatabases(metadataGetDatabasesAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetTablesAction: MetadataGetTablesAction = MetadataGetTablesAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDatabase("linkis_test_ind")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setSystem(system)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetTablesResult: MetadataGetTablesResult = metaDataClient.getTables(metadataGetTablesAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetColumnsAction = MetadataGetColumnsAction.builder()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setUser(user)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDataSourceId(dataSourceId)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setDatabase("linkis_test_ind")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setSystem(system)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .setTable("test")</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> .build()</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> val metadataGetColumnsResult: MetadataGetColumnsResult = metaDataClient.getColumns(metadataGetColumnsAction)</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain"> }</span><br></span><span class="token-line" style="color:#F8F8F2"><span class="token plain">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg class="copyButtonIcon_y97N" viewBox="0 0 24 24"><path d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg class="copyButtonSuccessIcon_LjdS" viewBox="0 0 24 24"><path d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><footer class="theme-doc-footer docusaurus-mt-lg"><div class="theme-doc-footer-edit-meta-row row"><div class="col"><a href="https://github.com/apache/linkis-website/edit/dev/versioned_docs/version-1.1.3/deployment/start-metadatasource.md" target="_blank" rel="noreferrer noopener" class="theme-edit-this-page"><svg fill="currentColor" height="20" width="20" viewBox="0 0 40 40" class="iconEdit_Z9Sw" aria-hidden="true"><g><path d="m34.5 11.7l-3 3.1-6.3-6.3 3.1-3q0.5-0.5 1.2-0.5t1.1 0.5l3.9 3.9q0.5 0.4 0.5 1.1t-0.5 1.2z m-29.5 17.1l18.4-18.5 6.3 6.3-18.4 18.4h-6.3v-6.2z"></path></g></svg>Edit this page</a></div><div class="col lastUpdated_vwxv"></div></div></footer></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/docs/1.1.3/deployment/involve-skywalking-into-linkis"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Involve SkyWaling into Linkis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/docs/1.1.3/deployment/deploy-linkis-without-hdfs"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Deploy Linkis without HDFS</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-data-source-function-introduction" class="table-of-contents__link toc-highlight">1. Data source function introduction</a><ul><li><a href="#11-concept" class="table-of-contents__link toc-highlight">1.1 Concept</a></li><li><a href="#12-three-main-modules" class="table-of-contents__link toc-highlight">1.2 Three main modules</a></li><li><a href="#13-processing-logic" class="table-of-contents__link toc-highlight">1.3 Processing logic</a></li><li><a href="#13-source-module-directory-structure" class="table-of-contents__link toc-highlight">1.3 Source module directory structure</a></li><li><a href="#14-installation-package-directory-structure" class="table-of-contents__link toc-highlight">1.4 Installation package directory structure</a></li><li><a href="#15-configuration-parameters" class="table-of-contents__link toc-highlight">1.5 Configuration Parameters</a></li></ul></li><li><a href="#2-enable-data-source-function" class="table-of-contents__link toc-highlight">2. Enable data source function</a></li><li><a href="#3-use-of-data-sources" class="table-of-contents__link toc-highlight">3. Use of data sources</a><ul><li><a href="#31-mysql-data-source" class="table-of-contents__link toc-highlight">3.1 Mysql data source</a></li><li><a href="#32-hive-data-source" class="table-of-contents__link toc-highlight">3.2 Hive data source</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Linkis</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/docs/latest/about/introduction">Document</a></li><li class="footer__item"><a class="footer__link-item" href="/faq/main">FAQ</a></li><li class="footer__item"><a href="https://github.com/apache/linkis/releases" target="_blank" rel="noopener noreferrer" class="footer__link-item">Releases</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/apache/linkis" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub</a></li><li class="footer__item"><a href="https://github.com/apache/linkis/issues" target="_blank" rel="noopener noreferrer" class="footer__link-item">Issue Tracker</a></li><li class="footer__item"><a href="https://github.com/apache/linkis/pulls" target="_blank" rel="noopener noreferrer" class="footer__link-item">Pull Requests</a></li></ul></div><div class="col footer__col"><div class="footer__title">Subscribe Mailing List</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/community/how-to-subscribe">How to Subscribe</a></li><li class="footer__item"><a href="mailto:dev-subscribe@linkis.apache.org" target="_blank" rel="noopener noreferrer" class="footer__link-item">Subscribe Mail</a></li><li class="footer__item"><a href="https://lists.apache.org/list.html?dev@linkis.apache.org" target="_blank" rel="noopener noreferrer" class="footer__link-item">Mail Archive</a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright"><div><img style="height:50px" alt="Apache Software Foundation" src="/img/incubator-logo.svg"><p style="color: #999999; padding: 0 20px 30px;font-weight:400;text-align:left">Apache Linkis is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.</p><p></p> |
| <p style="padding: 0 20px 30px;color: #999999;font-weight: 400;"> Copyright © 2024 The Apache Software Foundation. Licensed under the Apache License, Version 2.0. Apache Linkis, Apache Incubator, Apache, the Apache feather logo, the Apache Linkis logo and the Apache Incubator project logo are trademarks of The Apache Software Foundation.</p> |
| <div></div></div></div></div></div></footer></div> |
| <script src="/assets/js/runtime~main.36b81271.js"></script> |
| <script src="/assets/js/main.f09c65a0.js"></script> |
| </body> |
| </html> |
