pyproject.toml

[build-system]
requires = ["poetry-core>=1.0.0,<2.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
build-backend = "poetry_dynamic_versioning.backend"


[tool.poetry]
name = "airbyte-cdk"
description = "A framework for writing Airbyte Connectors."
authors = ["Airbyte <contact@airbyte.io>"]
license = "MIT"
readme = "README.md"
homepage = "https://airbyte.com"
repository = "https://github.com/airbytehq/airbyte-python-cdk"
documentation = "https://docs.airbyte.io/"
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Developers",
    "Topic :: Scientific/Engineering",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Python :: 3.10",
]
keywords = ["airbyte", "connector-development-kit", "cdk"]

# Python CDK uses dynamic versioning: https://github.com/mtkennerly/poetry-dynamic-versioning
version = "0.0.0"  # Version will be calculated dynamically.

[tool.poetry-dynamic-versioning]
enable = true

[tool.poetry.dependencies]
python = ">=3.10,<3.13"
airbyte-protocol-models-dataclasses = "^0.15"

backoff = "*"
boltons = "^25.0.0"
cachetools = "*"
click = "^8.1.8"
dpath = "^2.1.6"
dunamai = "^1.22.0"
genson = "1.3.0"
google-cloud-secret-manager = "^2.17.0"
isodate = "~0.6.1"
Jinja2 = "~3.1.2"
jsonref = "~0.2"
jsonschema = "~4.17.3"  # 4.18 has some significant breaking changes: https://github.com/python-jsonschema/jsonschema/releases/tag/v4.18.0
packaging = "*"  # Transitive dependency used directly in code
pandas = "2.2.2"
psutil = "6.1.0" # TODO: Remove if unused
pydantic = "^2.7"
pyrate-limiter = "~3.1.0"
python-dateutil = "^2.9.0"
python-ulid = "^3.0.0"
PyYAML = "^6.0.1"
rapidfuzz = "^3.10.1" # TODO: Remove if unused
requests = "*"
requests_cache = "*"
rich-click = "^1.8.8"
typing-extensions = "*"  # Transitive dependency used directly in code
wcmatch = "10.0"
# Extras depedencies
avro = { version = ">=1.11.2,<1.13.0", optional = true }  # TODO: Move into dev dependencies if only used in tests
cohere = { version = "4.21", optional = true }
fastavro = { version = "~1.8.0", optional = true }
langchain = { version = "0.1.16", optional = true }
langchain_core = { version = "0.1.42", optional = true }
markdown = { version = "*", optional = true }  # TODO: Remove if unused
openai = { version = "0.27.9", extras = ["embeddings"], optional = true }  # Used indirectly by langchain library
pdf2image = { version = "1.16.3", optional = true }
"pdfminer.six" = { version = "20221105", optional = true }  # Used indirectly by unstructured library
pyarrow = { version = "^19.0.0", optional = true }
pytesseract = { version = "0.3.10", optional = true }  # Used indirectly by unstructured library
python-calamine = { version = "0.2.3", optional = true }  # TODO: Remove if unused
python-snappy = { version = "0.7.3", optional = true }  # TODO: remove if unused
tiktoken = { version = "0.8.0", optional = true }
nltk = { version = "3.9.1", optional = true }
# This will ensure that even when you run poetry install or pip install, the compatible version of numpy will always be chosen.
# airbyte-ci will try to install latest version when --use-local-cdk is used, resulting in the conflict.
numpy = "<2"
unstructured = { version = "0.10.27", extras = ["docx", "pptx"], optional = true }
"unstructured.pytesseract" = { version = ">=0.3.12", optional = true }
pyjwt = "^2.8.0"
cryptography = ">=44.0.0,<45.0.0"  # Constrained as transitive dependency due to a bug in newer versions
pytz = "2024.2"
pytest = {version = "^7", optional = true }
orjson = "^3.10.7"
serpyco-rs = "^1.10.2"
sqlalchemy = {version = "^2.0,!=2.0.36", optional = true }
xmltodict = ">=0.13,<0.15"
anyascii = "^0.3.2"
whenever = "^0.6.16"

[tool.poetry.group.dev.dependencies]
freezegun = "*"
mypy = "*"
asyncio = "3.4.3"
ruff = "^0.11.5"
pdoc = "^15.0.0"
poethepoet = "^0.24.2"
pyproject-flake8 = "^6.1.0"
pytest = "*"  # Version will be set by the 'dev' extra
pytest-memray = "^1.6.0"
pympler = "*"
pytest-cov = "*"
pytest-httpserver = "*"
pytest-mock = "*"
requests-mock = "*"
# Stubs packages for mypy typing
types-requests = "^2.32.0.20241016"
types-python-dateutil = "^2.9.0.20241003"
types-pyyaml = "^6.0.12.20240917"
types-cachetools = "^5.5.0.20240820"
deptry = "^0.23.0"

[tool.poetry.extras]
file-based = ["avro", "fastavro", "pyarrow", "unstructured", "pdf2image", "pdfminer.six", "unstructured.pytesseract", "pytesseract", "markdown", "python-calamine", "python-snappy"]
vector-db-based = ["langchain", "openai", "cohere", "tiktoken"]
sql = ["sqlalchemy"]
dev = ["pytest"]

[tool.poetry.scripts]

airbyte-cdk = "airbyte_cdk.cli.airbyte_cdk:cli"
source-declarative-manifest = "airbyte_cdk.cli.source_declarative_manifest:run"

[tool.isort]
skip = ["__init__.py"]  # TODO: Remove after this is fixed: https://github.com/airbytehq/airbyte-python-cdk/issues/12

# Ruff configuration moved to ruff.toml

[tool.poe.tasks]
# Installation
install = { shell = "poetry install --all-extras" }
lock = { shell = "poetry lock" }

# Build tasks
assemble = {cmd = "bin/generate-component-manifest-dagger.sh", help = "Generate component manifest files."}
build-package = {cmd = "poetry build", help = "Build the python package: source and wheels archives."}
build = {sequence = ["assemble", "build-package"], help = "Run all tasks to build the package."}

# Format check tasks
_format-check-ruff = {cmd = "poetry run ruff format --check .", help = "Check formatting with Ruff."}
_format-check-prettier = {cmd = "npx prettier . --check", help = "Check formatting with prettier."}
format-check = {sequence = ["_format-check-ruff", "_format-check-prettier"], help = "Check formatting for all file types.", ignore_fail = "return_non_zero"}

# Format fix tasks
_format-fix-ruff = {cmd = "poetry run ruff format .", help = "Format with Ruff."}
_format-fix-prettier = {cmd = "npx prettier . --write", help = "Format with prettier."}
format-fix = {sequence = ["_format-fix-ruff", "_format-fix-prettier"], help = "Format all file types.", ignore_fail = "return_non_zero"}

# Linting/Typing check tasks
_lint-ruff = {cmd = "poetry run ruff check .", help = "Lint with Ruff."}
type-check = {cmd = "poetry run mypy airbyte_cdk", help = "Type check modified files with mypy."}
lint = {sequence = ["_lint-ruff", "type-check"], help = "Lint all code. Includes type checking.", ignore_fail = "return_non_zero"}

# Lockfile check task
check-lockfile = {cmd = "poetry check", help = "Check the poetry lock file."}

# Linting/Typing fix tasks
lint-fix = { cmd = "poetry run ruff check --fix .", help = "Auto-fix any lint issues that Ruff can automatically resolve (excluding 'unsafe' fixes)." }
lint-fix-unsafe = { cmd = "poetry run ruff check --fix --unsafe-fixes .", help = "Lint-fix modified files, including 'unsafe' fixes. It is recommended to first commit any pending changes and then always manually review any unsafe changes applied." }

# ruff fix everything (ignoring non-Python fixes)
ruff-fix = { sequence = ["lint-fix", "_format-fix-ruff"],  help = "Lint-fix and format-fix all code." }

# Combined Check and Fix tasks

check-all = {sequence = ["lint", "format-check", "type-check", "check-lockfile"], help = "Lint, format, and type-check modified files.", ignore_fail = "return_non_zero"}
fix-all = {sequence = ["format-fix", "lint-fix"], help = "Lint-fix and format-fix modified files, ignoring unsafe fixes.", ignore_fail = "return_non_zero"}
fix-and-check = {sequence = ["fix-all", "check-all"], help = "Lint-fix and format-fix, then re-check to see if any issues remain.", ignore_fail = "return_non_zero"}

# PyTest tasks

pytest = {cmd = "poetry run coverage run -m pytest --durations=10", help = "Run all pytest tests."}
pytest-fast = {cmd = "poetry run coverage run -m pytest --durations=5 --exitfirst -m 'not flaky and not slow and not requires_creds'", help = "Run pytest tests, failing fast and excluding slow tests."}
unit-test-with-cov = {cmd = "poetry run pytest -s unit_tests --cov=airbyte_cdk --cov-report=term --cov-config ./pyproject.toml", help = "Run unit tests and create a coverage report."}

# Combined check tasks (other)

# TODO: find a version of the modified mypy check that works both locally and in CI.
check-local = {sequence = ["lint", "type-check", "check-lockfile", "unit-test-with-cov"], help = "Lint all code, type-check modified files, and run unit tests."}
check-ci = {sequence = ["check-lockfile", "build", "lint", "unit-test-with-cov"], help = "Build the package, lint and run unit tests. Does not include type-checking."}

# Build and check
pre-push = {sequence = ["build", "check-local"], help = "Run all build and check tasks."}

# API Docs with PDoc
docs-generate = {env = {PDOC_ALLOW_EXEC = "1"}, cmd = "python -m docs.generate run", help="Generate API documentation with PDoc."}
docs-preview = {shell = "poe docs-generate && open docs/generated/index.html", help="Generate API documentation with PDoc and then open the docs in the default web browser."}

[tool.check-wheel-contents]
# Quality control for Python wheel generation. Docs here:
# - https://github.com/jwodder/check-wheel-contents
ignore = [
  "W002"  # Duplicate files. (TODO: Fix the few duplicate files, mostly `__init__.py` files that have only copyright text.)
]

[tool.pytest.ini_options]
log_cli = true
log_cli_level = "INFO"
log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
log_cli_date_format = "%Y-%m-%d %H:%M:%S"
filterwarnings = [
    "ignore::airbyte_cdk.sources.source.ExperimentalClassWarning"
]

[tool.airbyte_ci]
python_versions = ["3.10", "3.11"]
optional_poetry_groups = ["dev"]
poetry_extras = ["file-based", "vector-db-based"]
poe_tasks = ["check-ci"]
mount_docker_socket = true

[tool.deptry]
exclude = [
  "bin",
  "docs",
  "unit_tests",
]

[tool.deptry.per_rule_ignores]
# This is a mapping of rules and package names to be ignored for that rule.

# DEP001: Project should not contain missing dependencies.
# https://deptry.com/rules-violations/#missing-dependencies-dep001
DEP001 = [
  # These are imported but not declared:
  "source_declarative_manifest",  # Imported only in dynamic import tests, not in main code
  "google"  # Imported via google.cloud.secretmanager_v1 which is provided by google-cloud-secret-manager
]

# DEP002: Project should not contain unused dependencies.
# https://deptry.com/rules-violations/#missing-dependencies-dep002
DEP002 = [
  "cryptography",  # Constrained as transitive dependency due to a bug in newer versions
  "google-cloud-secret-manager",  # Deptry can't detect that `google.cloud.secretmanager_v1` uses this package

  # TODO: Remove these dependencies if not needed:
  "avro",  # Only imported in `unit_tests` code
  "psutil",
  "rapidfuzz",
  "cohere",
  "markdown",
  "openai",
  "pdf2image",
  "pdfminer.six",
  "pytesseract",
  "python-calamine",
  "python-snappy",
  "tiktoken",
  "unstructured.pytesseract",
]

# DEP003: Project should not use transitive dependencies.
# https://deptry.com/rules-violations/#transitive-dependencies-dep003
DEP003 = [
  # Transitive dependencies that are imported directly
  "pydantic_core"  # Pydantic internals, no need to define as separate dependency
]

# DEP004: Project should not use development dependencies in non-development code.
# https://deptry.com/rules-violations/#misplaced-development-dependencies-dep004
DEP004 = [
  # The `airbyte_cdk.test.utils` module is main code.
  # TODO: These should probably be declared within a `tests` extra:
  "pytest",
  "requests_mock",
]