| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| import logging |
| import re |
| from pathlib import Path |
| from typing import Any, Dict |
| |
| import yaml |
| from pkg_resources import resource_isdir, resource_listdir, resource_stream |
| |
| from superset.commands.exceptions import CommandInvalidError |
| from superset.commands.importers.v1.examples import ImportExamplesCommand |
| from superset.commands.importers.v1.utils import METADATA_FILE_NAME |
| |
| _logger = logging.getLogger(__name__) |
| |
| YAML_EXTENSIONS = {".yaml", ".yml"} |
| |
| |
| def load_examples_from_configs( |
| force_data: bool = False, load_test_data: bool = False |
| ) -> None: |
| """ |
| Load all the examples inside superset/examples/configs/. |
| """ |
| contents = load_contents(load_test_data) |
| command = ImportExamplesCommand(contents, overwrite=True, force_data=force_data) |
| command.run() |
| |
| |
| def load_contents(load_test_data: bool = False) -> Dict[str, Any]: |
| """Traverse configs directory and load contents""" |
| root = Path("examples/configs") |
| resource_names = resource_listdir("superset", str(root)) |
| queue = [root / resource_name for resource_name in resource_names] |
| |
| contents: Dict[Path, str] = {} |
| while queue: |
| path_name = queue.pop() |
| test_re = re.compile(r"\.test\.|metadata\.yaml$") |
| |
| if resource_isdir("superset", str(path_name)): |
| queue.extend( |
| path_name / child_name |
| for child_name in resource_listdir("superset", str(path_name)) |
| ) |
| elif path_name.suffix.lower() in YAML_EXTENSIONS: |
| if load_test_data and test_re.search(str(path_name)) is None: |
| continue |
| contents[path_name] = ( |
| resource_stream("superset", str(path_name)).read().decode("utf-8") |
| ) |
| |
| return {str(path.relative_to(root)): content for path, content in contents.items()} |
| |
| |
| def load_configs_from_directory( |
| root: Path, overwrite: bool = True, force_data: bool = False |
| ) -> None: |
| """ |
| Load all the examples from a given directory. |
| """ |
| contents: Dict[str, str] = {} |
| queue = [root] |
| while queue: |
| path_name = queue.pop() |
| if path_name.is_dir(): |
| queue.extend(path_name.glob("*")) |
| elif path_name.suffix.lower() in YAML_EXTENSIONS: |
| with open(path_name) as fp: |
| contents[str(path_name.relative_to(root))] = fp.read() |
| |
| # removing "type" from the metadata allows us to import any exported model |
| # from the unzipped directory directly |
| metadata = yaml.load(contents.get(METADATA_FILE_NAME, "{}")) |
| if "type" in metadata: |
| del metadata["type"] |
| contents[METADATA_FILE_NAME] = yaml.dump(metadata) |
| |
| command = ImportExamplesCommand( |
| contents, overwrite=overwrite, force_data=force_data |
| ) |
| try: |
| command.run() |
| except CommandInvalidError as ex: |
| _logger.error("An error occurred: %s", ex.normalized_messages()) |