| {"version":3,"sources":["/home/madhan/Apache/git/atlas/docs/target/src/documents/Hook/HookSqoop.md","/home/madhan/Apache/git/atlas/docs/target/theme/styles/styled-colors.js"],"names":["layoutProps","MDXContent","components","props","mdxType","parentName","wrapLines","language","style","theme","isMDXComponent","dark","hljs","color"],"mappings":"ykBAWMA,EAAc,GAIL,SAASC,EAAW,GAG/B,IAFFC,EAAU,EAAVA,WACGC,EAAK,iBAER,OAAO,cALS,UAKC,iBAAKH,EAAiBG,EAAK,CAAED,WAAYA,EAAYE,QAAQ,cAE5E,oBACE,GAAM,sCAAoC,sCAE5C,oBACE,GAAM,eAAa,eAErB,oEACA,wBACE,oBAAIC,WAAW,MAAI,gBACjB,oBAAIA,WAAW,MACb,oBAAIA,WAAW,MAAI,gBACjB,oBAAIA,WAAW,MACb,oBAAIA,WAAW,MAAI,wBACnB,oBAAIA,WAAW,MAAI,oIAGvB,oBAAIA,WAAW,MAAI,oBACjB,oBAAIA,WAAW,MACb,oBAAIA,WAAW,MAAI,wBACnB,oBAAIA,WAAW,MAAI,oGAK3B,oBAAIA,WAAW,MAAI,cACjB,oBAAIA,WAAW,MACb,oBAAIA,WAAW,MAAI,uBACjB,oBAAIA,WAAW,MACb,oBAAIA,WAAW,MAAI,kCAGvB,oBAAIA,WAAW,MAAI,sBACjB,oBAAIA,WAAW,MACb,oBAAIA,WAAW,MAAI,+CAM7B,oKACA,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,QAAQC,MAAOC,IAAYL,QAAQ,qBAAmB,4YAInG,oBACE,GAAM,cAAY,cAEpB,oPAEA,qFACA,sFACA,wBACE,oBAAIC,WAAW,MAAI,wBAA0B,4BAAYA,WAAW,MAAI,gBAA8B,6CAExG,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,QAAQC,MAAOC,IAAYL,QAAQ,qBAAmB,yIAMnG,wBACE,oBAAIC,WAAW,MAAI,2DACnB,oBAAIA,WAAW,MAAI,iDACnB,oBAAIA,WAAW,MAAI,2FAA8F,4BAAYA,WAAW,MAAI,mBAAiC,eAC7K,oBAAIA,WAAW,MAAI,QAAU,4BAAYA,WAAW,MAAI,gBAA8B,6DAA8D,4BAAYA,WAAW,MAAI,gBAA8B,KAC7M,oBAAIA,WAAW,MAAI,QAAU,4BAAYA,WAAW,MAAI,mBAAiC,mCAE3F,qIACA,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,QAAQC,MAAOC,IAAYL,QAAQ,qBAAmB,wzBAUnG,gOAA6M,mBAAGC,WAAW,IACvN,KAAQ,0DAAwD,2BAEpE,oBACE,GAAM,SAAO,SAEf,wBACE,oBAAIA,WAAW,MAAI,2EACjB,oBAAIA,WAAW,MACb,oBAAIA,WAAW,MAAI,kBAK5B,qLAEDJ,EAAWS,gBAAiB,G,+DCnH5B,iFAqBAC,IAAKC,KAAKC,MAAQ,UACHF,MAAI","file":"static/js/documents-hook-hook-sqoop.de797b78.js","sourcesContent":["\nimport React from 'react'\nimport { mdx } from '@mdx-js/react'\n\n/* @jsxRuntime classic */\n/* @jsx mdx */\nimport themen from 'theme/styles/styled-colors';\nimport * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';\nimport SyntaxHighlighter from 'react-syntax-highlighter';\n\n\nconst layoutProps = {\n \n};\nconst MDXLayout = \"wrapper\"\nexport default function MDXContent({\n components,\n ...props\n}) {\n return <MDXLayout {...layoutProps} {...props} components={components} mdxType=\"MDXLayout\">\n\n <h1 {...{\n \"id\": \"apache-atlas-hook-for-apache-sqoop\"\n }}>{`Apache Atlas Hook for Apache Sqoop`}</h1>\n <h2 {...{\n \"id\": \"sqoop-model\"\n }}>{`Sqoop Model`}</h2>\n <p>{`Sqoop model includes the following types:`}</p>\n <ul>\n <li parentName=\"ul\">{`Entity types:`}\n <ul parentName=\"li\">\n <li parentName=\"ul\">{`sqoop_process`}\n <ul parentName=\"li\">\n <li parentName=\"ul\">{`super-types: Process`}</li>\n <li parentName=\"ul\">{`attributes: qualifiedName, name, description, owner, inputs, outputs, operation, commandlineOpts, startTime, endTime, userName`}</li>\n </ul>\n </li>\n <li parentName=\"ul\">{`sqoop_dbdatastore`}\n <ul parentName=\"li\">\n <li parentName=\"ul\">{`super-types: DataSet`}</li>\n <li parentName=\"ul\">{`attributes: qualifiedName, name, description, owner, dbStoreType, storeUse, storeUri, source`}</li>\n </ul>\n </li>\n </ul>\n </li>\n <li parentName=\"ul\">{`Enum types:`}\n <ul parentName=\"li\">\n <li parentName=\"ul\">{`sqoop_operation_type`}\n <ul parentName=\"li\">\n <li parentName=\"ul\">{`values: IMPORT, EXPORT, EVAL`}</li>\n </ul>\n </li>\n <li parentName=\"ul\">{`sqoop_dbstore_usage`}\n <ul parentName=\"li\">\n <li parentName=\"ul\">{`values: TABLE, QUERY, PROCEDURE, OTHER`}</li>\n </ul>\n </li>\n </ul>\n </li>\n </ul>\n <p>{`Sqoop entities are created and de-duped in Atlas using unique attribute qualifiedName, whose value should be formatted as detailed below.`}</p>\n <SyntaxHighlighter wrapLines={true} language=\"shell\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`sqoop_process.qualifiedName: sqoop <operation> --connect <url> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>]\nsqoop_dbdatastore.qualifiedName: <storeType> --url <storeUri> {[--table <tableName>] || [--database <databaseName>]} [--query <storeQuery>] --hive-<operation> --hive-database <databaseName> [--hive-table <tableName>] --hive-cluster <clusterName>`}\n </SyntaxHighlighter>\n <h2 {...{\n \"id\": \"sqoop-hook\"\n }}>{`Sqoop Hook`}</h2>\n <p>{`Sqoop added a SqoopJobDataPublisher that publishes data to Atlas after completion of import Job. Today, only hiveImport is supported in SqoopHook.\nThis is used to add entities in Atlas using the model detailed above.`}</p>\n <p>{`Follow the instructions below to setup Atlas hook in Hive:`}</p>\n <p>{`Add the following properties to enable Atlas hook in Sqoop:`}</p>\n <ul>\n <li parentName=\"ul\">{`Set-up Atlas hook in `}<inlineCode parentName=\"li\">{`<sqoop-conf>`}</inlineCode>{`/sqoop-site.xml by adding the following:`}</li>\n </ul>\n <SyntaxHighlighter wrapLines={true} language=\"shell\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`<property>\n <name>sqoop.job.data.publish.class</name>\n <value>org.apache.atlas.sqoop.hook.SqoopHook</value>\n </property>`}\n </SyntaxHighlighter>\n <ul>\n <li parentName=\"ul\">{`untar apache-atlas-\\${project.version}-sqoop-hook.tar.gz`}</li>\n <li parentName=\"ul\">{`cd apache-atlas-sqoop-hook-\\${project.version}`}</li>\n <li parentName=\"ul\">{`Copy entire contents of folder apache-atlas-sqoop-hook-\\${project.version}/hook/sqoop to `}<inlineCode parentName=\"li\">{`<atlas package>`}</inlineCode>{`/hook/sqoop`}</li>\n <li parentName=\"ul\">{`Copy `}<inlineCode parentName=\"li\">{`<atlas-conf>`}</inlineCode>{`/atlas-application.properties to the sqoop conf directory `}<inlineCode parentName=\"li\">{`<sqoop-conf>`}</inlineCode>{`/`}</li>\n <li parentName=\"ul\">{`Link `}<inlineCode parentName=\"li\">{`<atlas package>`}</inlineCode>{`/hook/sqoop/*.jar in sqoop lib`}</li>\n </ul>\n <p>{`The following properties in atlas-application.properties control the thread pool and notification details:`}</p>\n <SyntaxHighlighter wrapLines={true} language=\"shell\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`atlas.hook.sqoop.synchronous=false # whether to run the hook synchronously. false is recommended to avoid delays in Sqoop operation completion. Default: false\natlas.hook.sqoop.numRetries=3 # number of retries for notification failure. Default: 3\natlas.hook.sqoop.queueSize=10000 # queue size for the threadpool. Default: 10000\natlas.cluster.name=primary # clusterName to use in qualifiedName of entities. Default: primary\natlas.kafka.zookeeper.connect= # Zookeeper connect URL for Kafka. Example: localhost:2181\natlas.kafka.zookeeper.connection.timeout.ms=30000 # Zookeeper connection timeout. Default: 30000\natlas.kafka.zookeeper.session.timeout.ms=60000 # Zookeeper session timeout. Default: 60000\natlas.kafka.zookeeper.sync.time.ms=20 # Zookeeper sync time. Default: 20`}\n </SyntaxHighlighter>\n <p>{`Other configurations for Kafka notification producer can be specified by prefixing the configuration name with \"atlas.kafka.\". For list of configuration supported by Kafka producer, please refer to `}<a parentName=\"p\" {...{\n \"href\": \"http://kafka.apache.org/documentation/#producerconfigs\"\n }}>{`Kafka Producer Configs`}</a></p>\n <h2 {...{\n \"id\": \"notes\"\n }}>{`NOTES`}</h2>\n <ul>\n <li parentName=\"ul\">{`Only the following sqoop operations are captured by sqoop hook currently`}\n <ul parentName=\"li\">\n <li parentName=\"ul\">{`hiveImport`}</li>\n </ul>\n </li>\n </ul>\n </MDXLayout>;\n}\n;\nMDXContent.isMDXComponent = true;","/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { dark } from \"react-syntax-highlighter/dist/esm/styles/hljs\";\n\n//dark[\"powershell\"][\"color\"] = \"#37bb9b\";\ndark.hljs.color = \"#37bb9b\";\nexport default dark;"],"sourceRoot":""} |