| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| [tool.poetry] |
| name = "pyiceberg" |
| version = "0.6.1" |
| readme = "README.md" |
| homepage = "https://py.iceberg.apache.org/" |
| repository = "https://github.com/apache/iceberg-python" |
| description = "Apache Iceberg is an open table format for huge analytic datasets" |
| authors = ["Apache Software Foundation <dev@iceberg.apache.org>"] |
| license = "Apache License 2.0" |
| classifiers = [ |
| "License :: OSI Approved :: Apache Software License", |
| "Operating System :: OS Independent", |
| "Programming Language :: Python :: 3.8", |
| "Programming Language :: Python :: 3.9", |
| "Programming Language :: Python :: 3.10", |
| "Programming Language :: Python :: 3.11" |
| ] |
| packages = [ |
| { include = "pyiceberg" }, |
| { from = "vendor", include = "fb303" }, |
| { from = "vendor", include = "hive_metastore" }, |
| { include = "tests", format = "sdist" }, |
| { include = "Makefile", format = "sdist" }, |
| { include = "NOTICE", format = ["sdist", "wheel"] } |
| ] |
| include = [ |
| { path = "dev", format = "sdist" }, |
| { path = "pyiceberg/**/*.so", format = "wheel" }, |
| { path = "pyiceberg/**/*.pyd", format = "wheel" }, |
| ] |
| |
| [tool.poetry.dependencies] |
| python = "^3.8" |
| mmhash3 = ">=3.0.0,<4.0.0" |
| requests = ">=2.20.0,<3.0.0" |
| click = ">=7.1.1,<9.0.0" |
| rich = ">=10.11.0,<14.0.0" |
| strictyaml = ">=1.7.0,<2.0.0" # CVE-2020-14343 was fixed in 5.4. |
| pydantic = ">=2.0,<3.0,!=2.4.0,!=2.4.1" # 2.4.0, 2.4.1 has a critical bug |
| sortedcontainers = "2.4.0" |
| fsspec = ">=2023.1.0,<2024.1.0" |
| pyparsing = ">=3.1.0,<4.0.0" |
| zstandard = ">=0.13.0,<1.0.0" |
| pyarrow = { version = ">=9.0.0,<16.0.0", optional = true } |
| pandas = { version = ">=1.0.0,<3.0.0", optional = true } |
| duckdb = { version = ">=0.5.0,<1.0.0", optional = true } |
| ray = { version = ">=2.0.0,<2.10.0", optional = true } |
| python-snappy = { version = ">=0.6.0,<1.0.0", optional = true } |
| thrift = { version = ">=0.13.0,<1.0.0", optional = true } |
| mypy-boto3-glue = { version = ">=1.28.18", optional = true } |
| boto3 = { version = ">=1.24.59", optional = true } |
| s3fs = { version = ">=2023.1.0,<2024.1.0", optional = true } |
| adlfs = { version = ">=2023.1.0,<2024.3.0", optional = true } |
| gcsfs = { version = ">=2023.1.0,<2024.1.0", optional = true } |
| psycopg2-binary = { version = ">=2.9.6", optional = true } |
| sqlalchemy = { version = "^2.0.18", optional = true } |
| getdaft = { version = ">=0.2.12", optional = true } |
| |
| [tool.poetry.dev-dependencies] |
| pytest = "7.4.4" |
| pytest-checkdocs = "2.10.1" |
| pytest-lazy-fixture = "0.6.3" |
| pre-commit = "3.5.0" |
| fastavro = "1.9.4" |
| coverage = { version = "^7.4.1", extras = ["toml"] } |
| requests-mock = "1.11.0" |
| moto = { version = "^5.0.1", extras = ["server"] } |
| typing-extensions = "4.9.0" |
| pytest-mock = "3.12.0" |
| pyspark = "3.5.0" |
| cython = "3.0.8" |
| docutils = "!=0.21" |
| |
| [[tool.mypy.overrides]] |
| module = "pytest_mock.*" |
| ignore_missing_imports = true |
| |
| [tool.poetry.scripts] |
| pyiceberg = "pyiceberg.cli.console:run" |
| |
| [build-system] |
| requires = ["poetry-core>=1.0.0", "wheel", "Cython>=3.0.0", "setuptools"] |
| build-backend = "poetry.core.masonry.api" |
| |
| [tool.poetry.build] |
| generate-setup-file = false |
| script = "build-module.py" |
| |
| [tool.poetry.extras] |
| pyarrow = ["pyarrow"] |
| pandas = ["pandas", "pyarrow"] |
| duckdb = ["duckdb", "pyarrow"] |
| ray = ["ray", "pyarrow", "pandas"] |
| daft = ["getdaft"] |
| snappy = ["python-snappy"] |
| hive = ["thrift"] |
| s3fs = ["s3fs"] |
| glue = ["boto3", "mypy-boto3-glue"] |
| adlfs = ["adlfs"] |
| dynamodb = ["boto3"] |
| zstandard = ["zstandard"] |
| sql-postgres = ["sqlalchemy", "psycopg2-binary"] |
| sql-sqlite = ["sqlalchemy"] |
| gcsfs = ["gcsfs"] |
| |
| [tool.pytest.ini_options] |
| markers = [ |
| "unmarked: marks a test as a unittest", |
| "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint args)", |
| "adlfs: marks a test as requiring access to adlfs compliant storage (use with --adlfs.account-name, --adlfs.account-key, and --adlfs.endpoint args)", |
| "integration: marks integration tests against Apache Spark", |
| "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint)" |
| ] |
| |
| # Turns a warning into an error |
| #filterwarnings = [ |
| # "error" |
| #] |
| |
| [tool.black] |
| line-length = 130 |
| target-version = ['py38'] |
| |
| [tool.pycln] |
| all = true |
| |
| [tool.mypy] |
| mypy_path = "python" |
| no_implicit_optional = true |
| namespace_packages = false |
| warn_redundant_casts = true |
| warn_unreachable = true |
| warn_unused_ignores = true |
| disallow_any_generics = true |
| disallow_untyped_defs = true |
| |
| [[tool.mypy.overrides]] |
| module = "pyarrow.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "pandas.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "snappy.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "zstandard.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "pydantic.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "pydantic_core.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "pytest.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "fastavro.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "mmh3.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "hive_metastore.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "thrift.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "requests_mock.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "click.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "rich.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "fsspec.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "s3fs.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "azure.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "adlfs.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "gcsfs.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "packaging.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "tests.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "boto3" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "botocore.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "mypy_boto3_glue.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "moto" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "aiobotocore.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "aiohttp.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "duckdb.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "ray.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "daft.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "pyparsing.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "pyspark.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "strictyaml.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "sortedcontainers.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "numpy.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "sqlalchemy.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "Cython.*" |
| ignore_missing_imports = true |
| |
| [[tool.mypy.overrides]] |
| module = "setuptools.*" |
| ignore_missing_imports = true |
| |
| [tool.coverage.run] |
| source = ['pyiceberg/'] |
| |
| [tool.ruff] |
| src = ['pyiceberg','tests'] |
| extend-exclude = ["dev/provision.py"] |
| select = [ |
| "E", # pycodestyle |
| "W", # pycodestyle |
| "F", # Pyflakes |
| "B", # flake8-bugbear |
| "PIE", # flake8-pie |
| "C4", # flake8-comprehensions |
| "I", # isort |
| "UP", # pyupgrade |
| ] |
| ignore = ["E501","E203","B024","B028","UP037"] |
| |
| # Allow autofix for all enabled rules (when `--fix`) is provided. |
| fixable = ["ALL"] |
| unfixable = [] |
| |
| # Exclude a variety of commonly ignored directories. |
| exclude = [ |
| ".bzr", |
| ".direnv", |
| ".eggs", |
| ".git", |
| ".git-rewrite", |
| ".hg", |
| ".mypy_cache", |
| ".nox", |
| ".pants.d", |
| ".pytype", |
| ".ruff_cache", |
| ".svn", |
| ".tox", |
| ".venv", |
| "__pypackages__", |
| "_build", |
| "buck-out", |
| "build", |
| "dist", |
| "node_modules", |
| "venv", |
| ] |
| per-file-ignores = {} |
| # Ignore _all_ violations. |
| # Same as Black. |
| line-length = 130 |
| |
| # Allow unused variables when underscore-prefixed. |
| dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" |
| |
| [tool.ruff.pyupgrade] |
| # Preserve types, even if a file imports `from __future__ import annotations`. |
| keep-runtime-typing = true |
| |
| [tool.ruff.isort] |
| detect-same-package = true |
| lines-between-types = 0 |
| known-first-party = ["pyiceberg", "tests"] |
| section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"] |
| |
| [tool.ruff.format] |
| quote-style = "preserve" |