| {"version":3,"sources":["/home/madhan/Apache/git/atlas/docs/target/src/documents/Import-Export/ImportAPIOptions.md","/home/madhan/Apache/git/atlas/docs/target/theme/styles/styled-colors.js"],"names":["layoutProps","MDXContent","components","props","mdxType","parentName","wrapLines","language","style","theme","isMDXComponent","dark","hljs","color"],"mappings":"ylBAWMA,EAAc,GAIL,SAASC,EAAW,GAG/B,IAFFC,EAAU,EAAVA,WACGC,EAAK,iBAER,OAAO,cALS,UAKC,iBAAKH,EAAiBG,EAAK,CAAED,WAAYA,EAAYE,QAAQ,cAE5E,oBACE,GAAM,sBAAoB,sBAE5B,8DAA2C,oBAAIC,WAAW,KAAG,WAAiB,6HAC9E,oBACE,GAAM,6BAA2B,6BAEnC,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,QAAQC,MAAOC,IAAYL,QAAQ,qBAAmB,0QAOnG,mEAAgD,oBAAIC,WAAW,KAAG,sBAA4B,QAC9F,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,OAAOC,MAAOC,IAAYL,QAAQ,qBAAmB,6BAMlG,oBACE,GAAM,WAAS,WAEjB,8EACA,wBACE,oBAAIC,WAAW,MAAI,mDACnB,oBAAIA,WAAW,MAAI,qDACnB,oBAAIA,WAAW,MAAI,sCACnB,oBAAIA,WAAW,MAAI,4BAErB,oBACE,GAAM,cAAY,cAEpB,gHACA,2JACA,kFACA,wBACE,oBAAIA,WAAW,MAAK,oBAAIA,WAAW,MAAI,aAAmB,gDAC1D,oBAAIA,WAAW,MAAK,oBAAIA,WAAW,MAAI,WAAiB,0KAE1D,mCACA,kFAA+D,oBAAIA,WAAW,KAAG,iBAAuB,6HACxG,iEAA8C,oBAAIA,WAAW,KAAG,sBAA4B,QAC5F,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,OAAOC,MAAOC,IAAYL,QAAQ,qBAAmB,mKAOlG,0CAAuB,mBAAGC,WAAW,IACjC,KAAQ,oDAAkD,cACvC,0DACvB,oBACE,GAAM,6BAA2B,6BAEnC,4OACA,0PACA,iEAA8C,oBAAIA,WAAW,KAAG,sBAA4B,QAC5F,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,QAAQC,MAAOC,IAAYL,QAAQ,qBAAmB,sFAOnG,iCAAc,oBAAIC,WAAW,KAAG,iBAAuB,8BAA+B,oBAAIA,WAAW,KAAG,sBAA4B,KACpI,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,OAAOC,MAAOC,IAAYL,QAAQ,qBAAmB,yDAOlG,qDACA,wBACE,oBAAIC,WAAW,MAAI,mFACnB,oBAAIA,WAAW,MAAI,6EACnB,oBAAIA,WAAW,MAAI,gCAAkC,oBAAIA,WAAW,MAAI,kCAAwC,sEAChH,oBAAIA,WAAW,MAAI,cAAgB,oBAAIA,WAAW,MAAI,sBAA4B,mBAClF,oBAAIA,WAAW,MAAI,oBAErB,oBACE,GAAM,sCAAoC,sCAE5C,mDAAgC,oBAAIA,WAAW,KAAG,sBAA4B,kEAC9E,uMACA,6JACA,mHAAgG,oBAAIA,WAAW,KAAG,QAAc,sFAAuF,oBAAIA,WAAW,KAAG,SAAe,wNACxP,sHACA,2BACE,uBAAOA,WAAW,SAChB,oBAAIA,WAAW,SACb,oBAAIA,WAAW,KACb,MAAS,MACR,wBAAQA,WAAW,MAAI,cAC1B,oBAAIA,WAAW,KACb,MAAS,MACR,wBAAQA,WAAW,MAAI,aAG9B,uBAAOA,WAAW,SAChB,oBAAIA,WAAW,SACb,oBAAIA,WAAW,KACb,MAAS,MAAI,qDAEf,oBAAIA,WAAW,KACb,MAAS,MAAI,qBAGjB,oBAAIA,WAAW,SACb,oBAAIA,WAAW,KACb,MAAS,MAAI,8DAEf,oBAAIA,WAAW,KACb,MAAS,MAAI,cAGjB,oBAAIA,WAAW,SACb,oBAAIA,WAAW,KACb,MAAS,MAAI,2EAEf,oBAAIA,WAAW,KACb,MAAS,MAAI,yEAC8D,yBAAM,4GAA6G,yBAAM,wFAAyF,yBAAM,yEAA0E,yBAAM,oGAI3X,iEAA8C,oBAAIA,WAAW,KAAG,sBAA4B,QAC5F,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,OAAOC,MAAOC,IAAYL,QAAQ,qBAAmB,+DAOlG,oBACE,GAAM,uDAAqD,uDAE7D,kHAA+F,oBAAIC,WAAW,KAAG,cAAoB,qDACrI,iEAA8C,oBAAIA,WAAW,KAAG,sBAA4B,QAC5F,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,OAAOC,MAAOC,IAAYL,QAAQ,qBAAmB,2EAOlG,uBAAG,oBAAIC,WAAW,KAAG,SACrB,cAAC,IAAiB,CAACC,WAAW,EAAMC,SAAS,OAAOC,MAAOC,IAAYL,QAAQ,qBAAmB,6NAMlG,oBACE,GAAM,0BAAwB,0BAEhC,yJACA,4FAAyE,oBAAIC,WAAW,KAAG,+BAAqC,mFAChI,yGACA,yJAEH,qMAEDJ,EAAWS,gBAAiB,G,+DC3L5B,iFAqBAC,IAAKC,KAAKC,MAAQ,UACHF,MAAI","file":"static/js/documents-import-export-import-api-options.d82d96e5.js","sourcesContent":["\nimport React from 'react'\nimport { mdx } from '@mdx-js/react'\n\n/* @jsxRuntime classic */\n/* @jsx mdx */\nimport themen from 'theme/styles/styled-colors';\nimport * as theme from 'react-syntax-highlighter/dist/esm/styles/hljs';\nimport SyntaxHighlighter from 'react-syntax-highlighter';\n\n\nconst layoutProps = {\n \n};\nconst MDXLayout = \"wrapper\"\nexport default function MDXContent({\n components,\n ...props\n}) {\n return <MDXLayout {...layoutProps} {...props} components={components} mdxType=\"MDXLayout\">\n\n <h1 {...{\n \"id\": \"import-api-options\"\n }}>{`Import API Options`}</h1>\n <p>{`Import API options are specified as `}<em parentName=\"p\">{`options`}</em>{` JSON. Since the API accepts multi-part form data, it is possible to specify multiple input streams within the CURL call.`}</p>\n <h3 {...{\n \"id\": \"examples-using-curl-calls\"\n }}>{`Examples Using CURL Calls`}</h3>\n <SyntaxHighlighter wrapLines={true} language=\"shell\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`curl -g -X POST -u adminuser:password -H \"Content-Type: multipart/form-data\"\n -H \"Cache-Control: no-cache\"\n -F request=@importOptions.json\n -F data=@quickStartDB.zip\n \"http://localhost:21000/api/atlas/admin/import\"`}\n </SyntaxHighlighter>\n <p>{`To use the defaults, set the contents of `}<em parentName=\"p\">{`importOptions.json`}</em>{` to:`}</p>\n <SyntaxHighlighter wrapLines={true} language=\"json\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`{\n \"options\": {\n }\n}`}\n </SyntaxHighlighter>\n <h3 {...{\n \"id\": \"options\"\n }}>{`Options`}</h3>\n <p>{`Following options are supported for Import process:`}</p>\n <ul>\n <li parentName=\"ul\">{`Specify transforms during the import operation.`}</li>\n <li parentName=\"ul\">{`Resume import by specifying starting entity guid.`}</li>\n <li parentName=\"ul\">{`Optionally import type definition.`}</li>\n <li parentName=\"ul\">{`Handling large imports.`}</li>\n </ul>\n <h4 {...{\n \"id\": \"transforms\"\n }}>{`Transforms`}</h4>\n <p>{`During the import process, the attribute value of the incoming entity can be changed.`}</p>\n <p>{`This is possible by specifying entity type and at attribute to be modified and then the manner in which it needs to be modified.`}</p>\n <p>{`Right now these are the transforms that can be applied:`}</p>\n <ul>\n <li parentName=\"ul\"><em parentName=\"li\">{`lowercase`}</em>{` Converts the attribute value to lower case.`}</li>\n <li parentName=\"ul\"><em parentName=\"li\">{`replace`}</em>{` This performs a string find and replace operation. It takes two parameters, the first is the string to search for and the next one is the string to replace it with.`}</li>\n </ul>\n <p>{`Example:`}</p>\n <p>{`The example below applies a couple of transforms to the `}<em parentName=\"p\">{`qualifiedName`}</em>{` attribute of hive_table. It converts the value to lower case, then searches for 'cl1', if found, replaces it with 'cl2'.`}</p>\n <p>{`To use the option, set the contents of `}<em parentName=\"p\">{`importOptions.json`}</em>{` to:`}</p>\n <SyntaxHighlighter wrapLines={true} language=\"json\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`{\n \"options\": {\n \"transforms\": {\"hive_table\": { \"qualifiedName\": [ replace:@cl1:@cl2 ] }, \"hive_db\": { \"qualifiedName\": [ replace:@cl1:@cl2 ] } }\n }\n}`}\n </SyntaxHighlighter>\n <p>{`Please refer to `}<a parentName=\"p\" {...{\n \"href\": \"https://issues.apache.org/jira/browse/ATLAS-1825\"\n }}>{`ATLAS-1825`}</a>{` for details scenarios when this option could be used.`}</p>\n <h4 {...{\n \"id\": \"start-guid-or-start-index\"\n }}>{`Start Guid or Start Index`}</h4>\n <p>{`When an import operation is in progress and the server goes down, it would be possible to resume import from the last successfully imported entity. This would allow the import to resume from where it left off.`}</p>\n <p>{`Server-side logging is improved to display the detail of the last successfully imported entity, this includes the index within the import list and the entity's guid. Either can be used to specify the point to resume import.`}</p>\n <p>{`To use the option, set the contents of `}<em parentName=\"p\">{`importOptions.json`}</em>{` to:`}</p>\n <SyntaxHighlighter wrapLines={true} language=\"shell\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`{\n \"options\": {\n \"startGuid\": \"bd97c78e-3fa5-4f9c-9f48-3683ca3d1fb1\"\n }\n}`}\n </SyntaxHighlighter>\n <p>{`To use `}<em parentName=\"p\">{`startPosition`}</em>{`, use the following in the `}<em parentName=\"p\">{`importOptions.json`}</em>{`:`}</p>\n <SyntaxHighlighter wrapLines={true} language=\"json\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`{\n \"options\": {\n \"startPosition\": \"332\"\n }\n}`}\n </SyntaxHighlighter>\n <p>{`Steps to use the behavior:`}</p>\n <ul>\n <li parentName=\"ul\">{`Start an import (using the CURL) that is fairly long, say about 1000# entities.`}</li>\n <li parentName=\"ul\">{`While the import is in progress, stop atlas server (using atlas_stop.py).`}</li>\n <li parentName=\"ul\">{`From the log file located at `}<em parentName=\"li\">{`/var/log/atlas/application.log`}</em>{` get the last successfully imported entity GUID or index position.`}</li>\n <li parentName=\"ul\">{`Update the `}<em parentName=\"li\">{`importOptions.json`}</em>{` with the guid.`}</li>\n <li parentName=\"ul\">{`Restart import.`}</li>\n </ul>\n <h4 {...{\n \"id\": \"optional-importing-type-definition\"\n }}>{`Optional Importing Type Definition`}</h4>\n <p>{`The output of Export has `}<em parentName=\"p\">{`atlas-typedef.json`}</em>{` that contains the type definitions for the entities exported.`}</p>\n <p>{`By default, (that is if no options are specified), the type definitions are imported and applied to the system being imported to. The entity import is performed after this.`}</p>\n <p>{`In some cases, you would not want to modify the type definitions. The import may be better off failing than the types be modified.`}</p>\n <p>{`This option allows for optionally importing of the type definition. The option is set to `}<em parentName=\"p\">{`true`}</em>{` by default, which means that type definition is imported. With this option set to `}<em parentName=\"p\">{`false`}</em>{`, type definitions present in the source will not be imported. In case of mismatch between the entities being imported the types present in the system where the import is being performed, the operation will fail.`}</p>\n <p>{`Table below enumerates the conditions that get addressed as part of type definition import:`}</p>\n <table>\n <thead parentName=\"table\">\n <tr parentName=\"thead\">\n <th parentName=\"tr\" {...{\n \"align\": null\n }}><strong parentName=\"th\">{`Condition`}</strong></th>\n <th parentName=\"tr\" {...{\n \"align\": null\n }}><strong parentName=\"th\">{`Action`}</strong></th>\n </tr>\n </thead>\n <tbody parentName=\"table\">\n <tr parentName=\"tbody\">\n <td parentName=\"tr\" {...{\n \"align\": null\n }}>{`Incoming type does not exist in the target system`}</td>\n <td parentName=\"tr\" {...{\n \"align\": null\n }}>{`Type is created.`}</td>\n </tr>\n <tr parentName=\"tbody\">\n <td parentName=\"tr\" {...{\n \"align\": null\n }}>{`Type to be imported and type in the target system are same`}</td>\n <td parentName=\"tr\" {...{\n \"align\": null\n }}>{`No change`}</td>\n </tr>\n <tr parentName=\"tbody\">\n <td parentName=\"tr\" {...{\n \"align\": null\n }}>{`Type to be imported and type in target system differ by some attributes`}</td>\n <td parentName=\"tr\" {...{\n \"align\": null\n }}>{`Target system type is updated to the attributes present in the source.`}<br />{` It is possible that the target system will have attributes in addition to the one present in the source.`}<br />{` In that case, the target system's type attributes will be a union of the attributes.`}<br />{` Attributes in target system will not be deleted to match the source. `}<br />{`If the type of the attribute differ, the import process will be aborted and exception logged.`}</td>\n </tr>\n </tbody>\n </table>\n <p>{`To use the option, set the contents of `}<em parentName=\"p\">{`importOptions.json`}</em>{` to:`}</p>\n <SyntaxHighlighter wrapLines={true} language=\"json\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`{\n \"options\": {\n \"updateTypeDefinition\": true\n }\n}`}\n </SyntaxHighlighter>\n <h4 {...{\n \"id\": \"specifying-file-to-be-imported-from-server-location\"\n }}>{`Specifying File to be Imported From Server Location`}</h4>\n <p>{`In a scenario where the file to be imported is present at a location on the server, the `}<em parentName=\"p\">{`importfile`}</em>{` API can be used. It behaves like the Import API.`}</p>\n <p>{`To use the option, set the contents of `}<em parentName=\"p\">{`importOptions.json`}</em>{` to:`}</p>\n <SyntaxHighlighter wrapLines={true} language=\"json\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`{\n \"options\": {\n \"fileName\": \"/root/fileToBeImported.zip\"\n }\n}`}\n </SyntaxHighlighter>\n <p><em parentName=\"p\">{`CURL`}</em></p>\n <SyntaxHighlighter wrapLines={true} language=\"json\" style={theme.dark} mdxType=\"SyntaxHighlighter\">\n {`curl -g -X POST -u adminuser:password -H \"Content-Type: application/json\"\n -H \"Cache-Control: no-cache\"\n -d r@importOptions.json\n \"http://localhost:21000/api/atlas/admin/importfile\"`}\n </SyntaxHighlighter>\n <h4 {...{\n \"id\": \"handling-large-imports\"\n }}>{`Handling Large Imports`}</h4>\n <p>{`By default, the Import Service stores all the data in memory. This may be limiting for ZIPs containing a large amount of data.`}</p>\n <p>{`To configure the temporary directory use the application property `}<em parentName=\"p\">{`atlas.import.temp.directory`}</em>{`. If this property is left blank, the default in-memory implementation is used.`}</p>\n <p>{`Please ensure that there is sufficient disk space available for the operation.`}</p>\n <p>{`The contents of the directory created as a backing store for the import operation will be erased after the operation is over.`}</p>\n </MDXLayout>;\n}\n;\nMDXContent.isMDXComponent = true;","/**\n * Licensed to the Apache Software Foundation (ASF) under one\n * or more contributor license agreements. See the NOTICE file\n * distributed with this work for additional information\n * regarding copyright ownership. The ASF licenses this file\n * to you under the Apache License, Version 2.0 (the\n * \"License\"); you may not use this file except in compliance\n * with the License. You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { dark } from \"react-syntax-highlighter/dist/esm/styles/hljs\";\n\n//dark[\"powershell\"][\"color\"] = \"#37bb9b\";\ndark.hljs.color = \"#37bb9b\";\nexport default dark;"],"sourceRoot":""} |