From 5e48dc1c70aacefe730c103864c5c320b8a135a0 Mon Sep 17 00:00:00 2001 From: Hussein Awala Date: Thu, 20 Aug 2026 12:50:28 +0200 Subject: [PATCH] Publish standalone documentation for Apache Airflow Mypy The independently released plugin needs version-aligned documentation and discoverable stable links instead of relying on the Airflow core release cadence. --- .github/workflows/publish-docs-to-s3.yml | 2 +- .../docs/howto/static-type-checking.rst | 56 +------------- dev/README_RELEASE_MYPY.md | 33 +++++++- dev/breeze/doc/ci/04_selective_checks.md | 4 +- dev/breeze/doc/images/output_build-docs.svg | 22 +++--- dev/breeze/doc/images/output_build-docs.txt | 2 +- ...release-management_add-back-references.svg | 22 +++--- ...release-management_add-back-references.txt | 2 +- ...output_release-management_publish-docs.svg | 22 +++--- ...output_release-management_publish-docs.txt | 2 +- .../output_workflow-run_publish-docs.svg | 22 +++--- .../output_workflow-run_publish-docs.txt | 2 +- .../commands/release_management_commands.py | 4 + .../src/airflow_breeze/global_constants.py | 15 ++++ .../utils/add_back_references.py | 3 + .../airflow_breeze/utils/docs_publisher.py | 3 + .../utils/publish_docs_to_s3.py | 1 + .../airflow_breeze/utils/selective_checks.py | 6 ++ dev/breeze/tests/test_add_back_references.py | 32 ++++++++ dev/breeze/tests/test_docs_publisher.py | 7 ++ dev/breeze/tests/test_publish_docs_to_s3.py | 2 + .../tests/test_release_management_commands.py | 6 ++ dev/breeze/tests/test_selective_checks.py | 7 ++ dev/mypy/docs/conf.py | 75 +++++++++++++++++++ dev/mypy/docs/index.rst | 75 +++++++++++++++++++ dev/mypy/docs/release_notes.rst | 21 ++++++ dev/mypy/docs/spelling_wordlist.txt | 20 +++++ dev/mypy/pyproject.toml | 16 +++- dev/registry/derive_wave_providers.py | 1 + .../tests/test_derive_wave_providers.py | 2 +- devel-common/pyproject.toml | 1 + .../docs_build/dev_index_template.html.jinja2 | 9 +++ .../sphinx_exts/docs_build/docs_builder.py | 3 + .../docs_build/test_docs_builder.py | 30 ++++++++ docs/README.md | 3 +- scripts/ci/docs/store_stable_versions.py | 5 ++ scripts/ci/prek/mypy_folder.py | 1 + .../ci/docs/test_store_stable_versions.py | 15 +++- uv.lock | 14 ++++ 39 files changed, 459 insertions(+), 109 deletions(-) create mode 100644 dev/breeze/tests/test_add_back_references.py create mode 100644 dev/mypy/docs/conf.py create mode 100644 dev/mypy/docs/index.rst create mode 100644 dev/mypy/docs/release_notes.rst create mode 100644 dev/mypy/docs/spelling_wordlist.txt create mode 100644 devel-common/tests/unit/sphinx_exts/docs_build/test_docs_builder.py diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index 7edd34e7d0757..061d5775fd284 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -153,7 +153,7 @@ jobs: run: | # Keep the non-provider list in sync with NON_PROVIDER_TOKENS in # dev/registry/derive_wave_providers.py. - NON_PROVIDER_TOKENS=("apache-airflow" "apache-airflow-ctl" "task-sdk" "helm-chart" "docker-stack") + NON_PROVIDER_TOKENS=("apache-airflow" "apache-airflow-ctl" "apache-airflow-mypy" "task-sdk" "helm-chart" "docker-stack") has_provider=false for token in $INCLUDE_DOCS; do is_non_provider=false diff --git a/airflow-core/docs/howto/static-type-checking.rst b/airflow-core/docs/howto/static-type-checking.rst index 0d8559434fce3..651a67bcf1b37 100644 --- a/airflow-core/docs/howto/static-type-checking.rst +++ b/airflow-core/docs/howto/static-type-checking.rst @@ -20,56 +20,6 @@ Static Type Checking for Dags ============================= -Airflow publishes a set of `mypy `_ plugins as a standalone, independently -versioned distribution: `apache-airflow-mypy `_. - -When to use it --------------- - -If you run ``mypy`` over your Dags, custom operators, or hooks, install the plugins to get accurate -results for Airflow-specific patterns that plain ``mypy`` cannot reason about and would otherwise report -as false positives. The plugins teach ``mypy`` about: - -* **Typed decorators** -- decorators that inject keyword arguments at runtime (for example - ``GoogleBaseHook.fallback_to_default_project_id``), so ``mypy`` does not flag those arguments as missing. -* **Operator outputs** -- the ``.output`` attribute of operators and the return value of ``@task``-decorated - functions (an ``XComArg``) are resolved to the underlying runtime type. This lets you wire a task's output - into a downstream task without spurious type errors: - - .. code-block:: python - - @task - def f(a: str) -> int: - return len(a) - - - @task - def g(b: int) -> None: ... - - - g(f("hello")) # mypy understands the output of f() is an int - -The package is entirely optional -- Airflow does not require it at runtime; it only improves the accuracy -of static type checking for Dag authors. - -Installation ------------- - -Install it alongside ``mypy``: - -.. code-block:: bash - - pip install apache-airflow-mypy - -The package follows `SemVer `_ and is released on its own cadence, so you can adopt it -independently of your Airflow version. - -Configuration -------------- - -Enable the plugins in your ``mypy`` configuration (``mypy.ini``, ``setup.cfg`` or ``pyproject.toml``): - -.. code-block:: ini - - [mypy] - plugins = airflow_mypy.plugins.decorators, airflow_mypy.plugins.outputs +Airflow publishes optional `mypy `_ plugins as the independently versioned +`apache-airflow-mypy `_ distribution. +See the plugin documentation for installation, configuration, supported Airflow patterns, and release notes. diff --git a/dev/README_RELEASE_MYPY.md b/dev/README_RELEASE_MYPY.md index 53138fd26927d..9df04a6d6a1fd 100644 --- a/dev/README_RELEASE_MYPY.md +++ b/dev/README_RELEASE_MYPY.md @@ -31,6 +31,7 @@ - [Add tags in git](#add-tags-in-git) - [Commit the source packages to Apache SVN repo](#commit-the-source-packages-to-apache-svn-repo) - [Publish the distributions to PyPI (release candidates)](#publish-the-distributions-to-pypi-release-candidates) + - [Publish release candidate documentation](#publish-release-candidate-documentation) - [Prepare voting email](#prepare-voting-email) - [Verify the release candidate by PMC members](#verify-the-release-candidate-by-pmc-members) - [Verify the release candidate by Contributors](#verify-the-release-candidate-by-contributors) @@ -39,6 +40,7 @@ - [Publish release to SVN](#publish-release-to-svn) - [Publish the packages to PyPI](#publish-the-packages-to-pypi) - [Add tags in git](#add-tags-in-git-1) + - [Publish documentation](#publish-documentation) - [Notify developers of release](#notify-developers-of-release) @@ -166,6 +168,18 @@ twine upload dist/apache_airflow_mypy-rc* Use a short-lived (throw-away) PyPI API token for the upload and delete it afterwards. +## Publish release candidate documentation + +Run the `Publish Docs to S3` workflow from the `apache/airflow` repository with: + +- **ref**: `apache-airflow-mypy-rc` +- **destination**: `staging` +- **include-docs**: `apache-airflow-mypy` + +After it completes, run the `Build docs` workflow from the `staging` branch of the `apache/airflow-site` +repository. Verify the release candidate documentation at +`https://airflow.staged.apache.org/docs/apache-airflow-mypy/stable/index.html` before starting the vote. + ## Prepare voting email Send a voting email to dev@airflow.apache.org with the following template: @@ -408,6 +422,20 @@ git tag -s apache-airflow-mypy- -m "Apache Airflow Mypy " git push origin apache-airflow-mypy- ``` +## Publish documentation + +Run the `Publish Docs to S3` workflow from the `apache/airflow` repository with: + +- **ref**: `apache-airflow-mypy-` +- **destination**: `live` +- **include-docs**: `apache-airflow-mypy` + +After it completes, run the `Build docs` workflow from the `main` branch of the `apache/airflow-site` +repository. Verify both the versioned and stable documentation URLs: + +- `https://airflow.apache.org/docs/apache-airflow-mypy//index.html` +- `https://airflow.apache.org/docs/apache-airflow-mypy/stable/index.html` + ## Notify developers of release Send an announcement email to dev@airflow.apache.org and announce@apache.org: @@ -422,8 +450,11 @@ Apache Airflow Mypy provides Mypy plugins for Apache Airflow to enhance type che The release is available at: https://pypi.org/project/apache-airflow-mypy// +Documentation: +https://airflow.apache.org/docs/apache-airflow-mypy/stable/index.html + Release notes: -https://github.com/apache/airflow/blob/main/dev/mypy/RELEASE_NOTES.rst +https://airflow.apache.org/docs/apache-airflow-mypy/stable/release_notes.html Installation: pip install apache-airflow-mypy diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 283d36636f913..2252c1cc7cf26 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -467,8 +467,8 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel * If there are some build dependencies changed (`hatch_build.py` and updated system dependencies in the `pyproject.toml` - then `upgrade to newer dependencies` is enabled. * If docs are build, the `docs-list-as-string` will determine which docs packages to build. This is based on - several criteria: if any of the airflow core, charts, docker-stack, providers files or docs have changed, - then corresponding packages are build (including cross-dependent providers). If any of the core files + several criteria: if any of the airflow core, charts, docker-stack, Apache Airflow Mypy, providers files or + docs have changed, then corresponding packages are built (including cross-dependent providers). If any of the core files changed, also providers docs are built because all providers depend on airflow docs. If any of the docs build python files changed or when build is "canary" type in main - all docs packages are built. diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 32179cfaf1269..a0eb454685574 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -241,17 +241,17 @@ Usage:                                                                                                                 breeze build-docs                                                                                                      [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   -datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   -ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk -| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | ts-sdk | vertica |     -vespa | weaviate | yandex | ydb | zendesk]...                                                                          +apache-airflow-mypy | apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink |       +apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | +cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging |  +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch |   +exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap |         +influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp |   +microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |    +pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      +sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram |   +teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                    Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index 1916a18fb3b61..301b3089dbb89 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -bd120b39a206e374a2867ca6020fba81 +6869c62bf6dd00ce2c54810da1e09c95 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 634c1cef692f4..915f449225024 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -150,17 +150,17 @@ Usage:                                                                                                                 breeze release-management add-back-references                                                                          [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   -datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   -ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk -| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | ts-sdk | vertica |     -vespa | weaviate | yandex | ydb | zendesk]...                                                                          +apache-airflow-mypy | apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink |       +apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | +cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging |  +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch |   +exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap |         +influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp |   +microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |    +pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      +sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram |   +teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                    Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index e5d9287fb408f..338338df1b533 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -541760eea56b065a0bd0573fba31bdb0 +da17572185eeb81b9714aa9514be1970 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index ba78a0c8a0b17..75477c8bf5793 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -189,17 +189,17 @@ Usage:                                                                                                                 breeze release-management publish-docs                                                                                 [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   -datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   -ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk -| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | ts-sdk | vertica |     -vespa | weaviate | yandex | ydb | zendesk]...                                                                          +apache-airflow-mypy | apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink |       +apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | +cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging |  +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch |   +exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap |         +influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp |   +microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |    +pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      +sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram |   +teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                    Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index 2113a9db3c8eb..cf48a3e8d3811 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -314fc7a431405a85bfe701e396ea09fb +ea0870d76dc087db72a64b430dc321fe diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 1eecdebc1ccfb..bbb753547ddf4 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -204,17 +204,17 @@ Usage:                                                                                                                 breeze workflow-run publish-docs                                                                                       [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | anthropic | apache-airflow | apache-airflow-ctl |   -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging | common.sql | databricks |   -datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook |   -ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | java-sdk -| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   -neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   -pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | -smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | ts-sdk | vertica |     -vespa | weaviate | yandex | ydb | zendesk]...                                                                          +apache-airflow-mypy | apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink |       +apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | +cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.dataquality | common.io | common.messaging |  +common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch |   +exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap |         +influxdb | informatica | java-sdk | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp |   +microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle |    +pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment |      +sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram |   +teradata | trino | ts-sdk | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                    Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index 46fa631c81775..d5c977ca9dcec 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -9845cbadb564c5fe56361e299dc1fb66 +f4e0b4ba219bf74c7f8935ef27799253 diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 84ddb9f6b4556..c4d9202e0eb39 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -102,6 +102,7 @@ MULTI_PLATFORM, SCHEMA_DESTINATION_LOCATIONS, UV_VERSION, + get_airflow_mypy_version, get_airflow_version, get_airflowctl_version, get_task_sdk_version, @@ -2044,6 +2045,9 @@ def get_package_version_possibly_from_stable_txt(package_name: str) -> str | Non if package_name == "apache-airflow-ctl": return get_airflowctl_version() + if package_name == "apache-airflow-mypy": + return get_airflow_mypy_version() + if package_name == "task-sdk": return get_task_sdk_version() diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index eeac2cf6bb2da..e0d64a8ada87f 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -33,6 +33,7 @@ AIRFLOW_CTL_SOURCES_PATH, AIRFLOW_ROOT_PATH, AIRFLOW_TASK_SDK_SOURCES_PATH, + MYPY_SOURCES_PATH, ) PUBLIC_AMD_RUNNERS = '["ubuntu-22.04"]' @@ -316,6 +317,7 @@ "task-sdk", "ts-sdk", "apache-airflow-ctl", + "apache-airflow-mypy", ] @@ -711,6 +713,19 @@ def get_airflowctl_version(): return airflowctl_version +def get_airflow_mypy_version(): + mypy_init_py_file = MYPY_SOURCES_PATH / "airflow_mypy" / "__init__.py" + mypy_version = "unknown" + with open(mypy_init_py_file) as init_file: + while line := init_file.readline(): + if "__version__ = " in line: + mypy_version = line.split()[2][1:-1] + break + if mypy_version == "unknown": + raise RuntimeError("Unable to determine Apache Airflow Mypy version") + return mypy_version + + def get_airflow_version(): airflow_init_py_file = AIRFLOW_CORE_SOURCES_PATH / "airflow" / "__init__.py" airflow_version = "unknown" diff --git a/dev/breeze/src/airflow_breeze/utils/add_back_references.py b/dev/breeze/src/airflow_breeze/utils/add_back_references.py index 318873f02f2da..75fd803fb3d7f 100644 --- a/dev/breeze/src/airflow_breeze/utils/add_back_references.py +++ b/dev/breeze/src/airflow_breeze/utils/add_back_references.py @@ -158,6 +158,9 @@ def start_generating_back_references( if "apache-airflow-ctl" in short_provider_ids: console_print("[info]Skipping airflowctl package. No back-reference needed.") short_provider_ids.remove("apache-airflow-ctl") + if "apache-airflow-mypy" in short_provider_ids: + console_print("[info]Skipping apache-airflow-mypy package. No back-reference needed.") + short_provider_ids.remove("apache-airflow-mypy") if short_provider_ids: for p in short_provider_ids: diff --git a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py index a27d7490bfc29..10dae33e9bd1b 100644 --- a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py +++ b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py @@ -20,6 +20,7 @@ import shutil from airflow_breeze.global_constants import ( + get_airflow_mypy_version, get_airflow_version, get_airflowctl_version, get_java_sdk_version, @@ -88,6 +89,8 @@ def _current_version(self): return chart_version() if self.package_name == "apache-airflow-ctl": return get_airflowctl_version() + if self.package_name == "apache-airflow-mypy": + return get_airflow_mypy_version() if self.package_name == "java-sdk": return get_java_sdk_version() raise SystemExit(f"Unsupported package: {self.package_name}") diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py index 972c1b472d6e6..507983583ce56 100644 --- a/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_to_s3.py @@ -34,6 +34,7 @@ NON_SHORT_NAME_PACKAGES = [ "apache-airflow", "apache-airflow-ctl", + "apache-airflow-mypy", "docker-stack", "helm-chart", "java-sdk", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 33dbaffc35c21..02c2e4bb9292a 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -367,6 +367,10 @@ def __hash__(self): r"^airflow-ctl/docs", r"^airflow-ctl/src/.*\.py$", r"^airflow-ctl/tests/.*\.py$", + r"^dev/mypy/docs/", + r"^dev/mypy/src/.*\.py$", + r"^dev/mypy/RELEASE_NOTES\.rst$", + r"^dev/mypy/pyproject\.toml$", r"^CHANGELOG\.txt", r"^airflow-core/src/airflow/config_templates/config\.yml", r"^chart/RELEASE_NOTES\.rst", @@ -1670,6 +1674,8 @@ def docs_list_as_string(self) -> str | None: packages.append("task-sdk") if any(file.startswith("airflow-ctl/") for file in self._files): packages.append("apache-airflow-ctl") + if any(file.startswith("dev/mypy/") for file in self._files): + packages.append("apache-airflow-mypy") if providers_affected: suspended = set(get_suspended_provider_ids()) for provider in providers_affected: diff --git a/dev/breeze/tests/test_add_back_references.py b/dev/breeze/tests/test_add_back_references.py new file mode 100644 index 0000000000000..fd6be5d04ef37 --- /dev/null +++ b/dev/breeze/tests/test_add_back_references.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from pathlib import Path +from unittest import mock + +from airflow_breeze.utils.add_back_references import start_generating_back_references + + +@mock.patch("airflow_breeze.utils.add_back_references.generate_back_references", autospec=True) +def test_mypy_docs_do_not_generate_provider_back_references(mock_generate_back_references): + package_ids = ["apache-airflow-mypy"] + + start_generating_back_references(Path("airflow-site"), package_ids) + + assert package_ids == [] + mock_generate_back_references.assert_not_called() diff --git a/dev/breeze/tests/test_docs_publisher.py b/dev/breeze/tests/test_docs_publisher.py index dc32b55256dac..83ba086acf098 100644 --- a/dev/breeze/tests/test_docs_publisher.py +++ b/dev/breeze/tests/test_docs_publisher.py @@ -114,6 +114,13 @@ def test_publish_java_sdk_version_falls_back_to_gradle_properties( assert (java_sdk_archive / "stable.txt").read_text() == "9.9.9\n" +def test_mypy_version_falls_back_to_package_source(monkeypatch): + monkeypatch.setattr(docs_publisher, "get_airflow_mypy_version", lambda: "0.2.0") + publisher = DocsPublisher(package_name="apache-airflow-mypy", output=None, verbose=False) + + assert publisher._current_version == "0.2.0" + + def test_publish_ts_sdk_version_falls_back_to_package_json(generated_path, airflow_site_dir, monkeypatch): monkeypatch.setattr(docs_publisher, "get_ts_sdk_version", lambda: "0.1.0-alpha.0") _stage_ts_sdk_docs(generated_path, version=None) diff --git a/dev/breeze/tests/test_publish_docs_to_s3.py b/dev/breeze/tests/test_publish_docs_to_s3.py index f0d419a43e958..b1b141269202f 100644 --- a/dev/breeze/tests/test_publish_docs_to_s3.py +++ b/dev/breeze/tests/test_publish_docs_to_s3.py @@ -60,6 +60,7 @@ def test_get_all_eligible_docs(self, mock_listdir): "apache-airflow-providers-apache-cassandra", "helm-chart", "apache-airflow-ctl", + "apache-airflow-mypy", ] self.publish_docs_to_s3.exclude_docs = "amazon docker-stack apache.kafka" @@ -71,6 +72,7 @@ def test_get_all_eligible_docs(self, mock_listdir): "apache-airflow-providers-apache-cassandra", "helm-chart", "apache-airflow-ctl", + "apache-airflow-mypy", ] ) diff --git a/dev/breeze/tests/test_release_management_commands.py b/dev/breeze/tests/test_release_management_commands.py index 63d2232bc650f..26209cbf54122 100644 --- a/dev/breeze/tests/test_release_management_commands.py +++ b/dev/breeze/tests/test_release_management_commands.py @@ -296,6 +296,12 @@ def test_get_package_version_possibly_from_stable_txt_for_java_sdk( assert get_package_version_possibly_from_stable_txt("java-sdk") == expected_version +def test_get_package_version_possibly_from_stable_txt_for_mypy(monkeypatch): + monkeypatch.setattr(release_management_commands, "get_airflow_mypy_version", lambda: "0.2.0") + + assert get_package_version_possibly_from_stable_txt("apache-airflow-mypy") == "0.2.0" + + @pytest.mark.parametrize( ("stable_txt_content", "expected_version"), [ diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index a91804ff4e832..9b953a10c66ee 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -3104,6 +3104,13 @@ def test_upgrade_to_newer_dependencies( }, id="Only Airflow docs changed", ), + pytest.param( + ("dev/mypy/docs/index.rst",), + { + "docs-list-as-string": "apache-airflow-mypy", + }, + id="Only Apache Airflow Mypy docs changed", + ), pytest.param( ("providers/celery/src/airflow/providers/celery/file.py",), {"docs-list-as-string": "celery cncf.kubernetes common.compat"}, diff --git a/dev/mypy/docs/conf.py b/dev/mypy/docs/conf.py new file mode 100644 index 0000000000000..713238f8e932b --- /dev/null +++ b/dev/mypy/docs/conf.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +from pathlib import Path + +from docs.utils.conf_constants import ( + AIRFLOW_FAVICON_PATH, + SPELLING_WORDLIST_PATH, + SPHINX_DESIGN_STATIC_PATH, + get_html_context, + get_html_sidebars, + get_html_theme_options, +) + +import airflow_mypy + +CONF_DIR = Path(__file__).parent.absolute() +PACKAGE_NAME = "apache-airflow-mypy" +PACKAGE_VERSION = airflow_mypy.__version__ + +os.environ["AIRFLOW_PACKAGE_NAME"] = PACKAGE_NAME + +project = "Apache Airflow Mypy" +version = PACKAGE_VERSION +release = PACKAGE_VERSION + +language = "en" +locale_dirs: list[str] = [] + +extensions = [ + "sphinx.ext.intersphinx", + "airflow_intersphinx", + "sphinxcontrib.spelling", +] + +html_theme = "sphinx_airflow_theme" +html_title = "Apache Airflow Mypy Documentation" +html_short_title = "Airflow Mypy" +html_favicon = AIRFLOW_FAVICON_PATH.as_posix() +html_static_path = [SPHINX_DESIGN_STATIC_PATH.as_posix()] +html_css_files = ["custom.css"] +html_sidebars = get_html_sidebars(PACKAGE_VERSION) +html_theme_options = get_html_theme_options() +conf_py_path = "/dev/mypy/docs/" +html_context = get_html_context(conf_py_path) +html_use_index = True +html_show_copyright = False + +intersphinx_mapping = { + "airflow": ("https://airflow.apache.org/docs/apache-airflow/stable/", None), +} + +spelling_show_suggestions = False +spelling_word_list_filename = [ + SPELLING_WORDLIST_PATH.as_posix(), + (CONF_DIR / "spelling_wordlist.txt").as_posix(), +] +spelling_ignore_importable_modules = True +spelling_ignore_contributor_names = True diff --git a/dev/mypy/docs/index.rst b/dev/mypy/docs/index.rst new file mode 100644 index 0000000000000..31e255344bb9d --- /dev/null +++ b/dev/mypy/docs/index.rst @@ -0,0 +1,75 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Apache Airflow Mypy plugins +=========================== + +``apache-airflow-mypy`` provides `mypy `_ plugins for Airflow-specific patterns. +The package is optional, independently versioned, and is not required to run Airflow. + +Use the plugins when type-checking Dags, custom operators, or hooks to avoid false positives that plain +``mypy`` cannot resolve. The plugins support: + +* **Typed decorators** -- decorators that inject keyword arguments at runtime, such as + ``GoogleBaseHook.fallback_to_default_project_id``. +* **Operator outputs** -- the ``.output`` attribute of operators and the return value of ``@task``-decorated + functions are resolved from ``XComArg`` to their underlying runtime type. + +Installation +------------ + +Install the package alongside ``mypy``: + +.. code-block:: bash + + pip install apache-airflow-mypy + +The package follows `SemVer `_ and can be upgraded independently of Airflow. + +Configuration +------------- + +Enable both plugins in ``mypy.ini``, ``setup.cfg``, or ``pyproject.toml``: + +.. code-block:: ini + + [mypy] + plugins = airflow_mypy.plugins.decorators, airflow_mypy.plugins.outputs + +For example, the output plugin lets ``mypy`` infer the return type of a TaskFlow task: + +.. code-block:: python + + @task + def count_characters(value: str) -> int: + return len(value) + + + @task + def report_count(count: int) -> None: ... + + + report_count(count_characters("Airflow")) + +Without the plugin, ``mypy`` sees an ``XComArg`` passed to ``report_count``. With the plugin enabled, it +understands that ``count_characters`` produces an ``int``. + +.. toctree:: + :hidden: + :caption: Reference + + release_notes diff --git a/dev/mypy/docs/release_notes.rst b/dev/mypy/docs/release_notes.rst new file mode 100644 index 0000000000000..d64d2f314b4e3 --- /dev/null +++ b/dev/mypy/docs/release_notes.rst @@ -0,0 +1,21 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Release Notes +============= + +.. include:: ../RELEASE_NOTES.rst diff --git a/dev/mypy/docs/spelling_wordlist.txt b/dev/mypy/docs/spelling_wordlist.txt new file mode 100644 index 0000000000000..171ae874ec648 --- /dev/null +++ b/dev/mypy/docs/spelling_wordlist.txt @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +mypy +TaskFlow +XComArg diff --git a/dev/mypy/pyproject.toml b/dev/mypy/pyproject.toml index 612e9e75674c0..f3f186738084a 100644 --- a/dev/mypy/pyproject.toml +++ b/dev/mypy/pyproject.toml @@ -52,7 +52,7 @@ dependencies = [ Documentation = "https://airflow.apache.org/docs/apache-airflow-mypy/stable/index.html" Downloads = "https://archive.apache.org/dist/airflow/airflow-mypy/" Homepage = "https://airflow.apache.org/" -"Release Notes" = "https://airflow.apache.org/docs/apache-airflow-mypy/stable/changelog.html" +"Release Notes" = "https://airflow.apache.org/docs/apache-airflow-mypy/stable/release_notes.html" "Slack Chat" = "https://s.apache.org/airflow-slack" "Source Code" = "https://github.com/apache/airflow" LinkedIn = "https://www.linkedin.com/company/apache-airflow/" @@ -78,6 +78,16 @@ path = "src/airflow_mypy/__init__.py" [tool.hatch.build.targets.wheel] packages = ["src/airflow_mypy"] +[dependency-groups] +# To build docs: +# +# uv run --group docs build-docs +# +docs = [ + "apache-airflow-core", + "apache-airflow-devel-common[docs]" +] + [tool.ruff] extend = "../../pyproject.toml" src = ["src"] @@ -128,3 +138,7 @@ showcontent = true [tool.uv] required-version = ">=0.11.8" + +[tool.uv.sources] +apache-airflow-core = { path = "../../airflow-core", editable = true } +apache-airflow-devel-common = { path = "../../devel-common", editable = true } diff --git a/dev/registry/derive_wave_providers.py b/dev/registry/derive_wave_providers.py index 374cf3e6ff148..9ce9d59c28fd9 100644 --- a/dev/registry/derive_wave_providers.py +++ b/dev/registry/derive_wave_providers.py @@ -59,6 +59,7 @@ "helm-chart", "docker-stack", "apache-airflow-ctl", + "apache-airflow-mypy", "task-sdk", } ) diff --git a/dev/registry/tests/test_derive_wave_providers.py b/dev/registry/tests/test_derive_wave_providers.py index 32a35d93b9520..08f011b52a056 100644 --- a/dev/registry/tests/test_derive_wave_providers.py +++ b/dev/registry/tests/test_derive_wave_providers.py @@ -90,7 +90,7 @@ def runner(*args): # Explicit with dot-to-hyphen conversion. ("common.ai", "any", "common-ai", False, ""), # Explicit with non-provider tokens stripped. - ("apache-airflow helm-chart docker-stack", "any", "", False, ""), + ("apache-airflow apache-airflow-mypy helm-chart docker-stack", "any", "", False, ""), # Explicit dedup. ("amazon amazon google", "any", "amazon google", False, ""), # Empty. diff --git a/devel-common/pyproject.toml b/devel-common/pyproject.toml index 17ad5220fddbd..5006e1fe2eba2 100644 --- a/devel-common/pyproject.toml +++ b/devel-common/pyproject.toml @@ -77,6 +77,7 @@ dependencies = [ "astroid>=4", "rich-click>=1.9.7", "click>=8.3.0", + "tabulate>=0.9.0", "docutils>=0.21", "pagefind>=1.5.0", "pagefind-bin>=1.5.0", diff --git a/devel-common/src/sphinx_exts/docs_build/dev_index_template.html.jinja2 b/devel-common/src/sphinx_exts/docs_build/dev_index_template.html.jinja2 index 4c990817d3888..9931753143f38 100644 --- a/devel-common/src/sphinx_exts/docs_build/dev_index_template.html.jinja2 +++ b/devel-common/src/sphinx_exts/docs_build/dev_index_template.html.jinja2 @@ -65,6 +65,15 @@ +
+
+

apache-airflow-mypy

+

+ Mypy plugins for Airflow-specific static type checking. +

+
+
+

Task SDK

diff --git a/devel-common/src/sphinx_exts/docs_build/docs_builder.py b/devel-common/src/sphinx_exts/docs_build/docs_builder.py index 147215a7353b2..12bf36e79a4fe 100644 --- a/devel-common/src/sphinx_exts/docs_build/docs_builder.py +++ b/devel-common/src/sphinx_exts/docs_build/docs_builder.py @@ -124,6 +124,8 @@ def _src_dir(self) -> Path: return (AIRFLOW_CONTENT_ROOT_PATH / "providers").joinpath(*package_paths) / "docs" if self.package_name == "apache-airflow-ctl": return AIRFLOW_CONTENT_ROOT_PATH / "airflow-ctl" / "docs" + if self.package_name == "apache-airflow-mypy": + return AIRFLOW_CONTENT_ROOT_PATH / "dev" / "mypy" / "docs" if self.package_name == "task-sdk": return AIRFLOW_CONTENT_ROOT_PATH / "task-sdk" / "docs" console.print(f"[red]Unknown package name: {self.package_name}") @@ -342,6 +344,7 @@ def get_available_packages(include_suspended: bool = False, short_form: bool = F *provider_names, "apache-airflow-providers", "apache-airflow-ctl", + "apache-airflow-mypy", "task-sdk", "helm-chart", "docker-stack", diff --git a/devel-common/tests/unit/sphinx_exts/docs_build/test_docs_builder.py b/devel-common/tests/unit/sphinx_exts/docs_build/test_docs_builder.py new file mode 100644 index 0000000000000..49434230c10b4 --- /dev/null +++ b/devel-common/tests/unit/sphinx_exts/docs_build/test_docs_builder.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from sphinx_exts.docs_build.code_utils import AIRFLOW_CONTENT_ROOT_PATH +from sphinx_exts.docs_build.docs_builder import AirflowDocsBuilder, get_available_packages + + +def test_mypy_docs_package_is_available(): + assert "apache-airflow-mypy" in get_available_packages() + + +def test_mypy_docs_source_directory(): + builder = AirflowDocsBuilder(package_name="apache-airflow-mypy") + + assert builder._src_dir == AIRFLOW_CONTENT_ROOT_PATH / "dev" / "mypy" / "docs" diff --git a/docs/README.md b/docs/README.md index 28ae32a1db8e0..02c4af4b5d85a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,7 @@ Documentation in separate distributions: * `chart/docs` - documentation for the Helm Chart * `task-sdk/docs` - documentation for Task SDK (new format not yet published) * `airflow-ctl/docs` - documentation for Airflow CLI +* `dev/mypy/docs` - documentation for the Apache Airflow Mypy plugins Documentation for a general overview and summaries not connected with any specific distribution: @@ -141,7 +142,7 @@ You can also specify whether documentation should be published to `live` or `sta The person who triggers the build (release manager) should specify the tag name of the docs to be published and the list of documentation packages to be published. Usually it is: -* Airflow: `apache-airflow docker-stack task-sdk apache-airflow-ctl` +* Airflow: `apache-airflow docker-stack task-sdk apache-airflow-ctl apache-airflow-mypy` * Helm chart: `helm-chart` * Providers: `provider_id1 provider_id2` or `all providers` if all providers should be published. diff --git a/scripts/ci/docs/store_stable_versions.py b/scripts/ci/docs/store_stable_versions.py index 91fb8247761b4..c00d7ab79f8f5 100755 --- a/scripts/ci/docs/store_stable_versions.py +++ b/scripts/ci/docs/store_stable_versions.py @@ -105,6 +105,11 @@ def get_package_version(package_name: str, airflow_root: Path) -> str | None: if package_name == "apache-airflow-ctl": return get_version_from_init_py(airflow_root / "airflow-ctl" / "src" / "airflowctl" / "__init__.py") + if package_name == "apache-airflow-mypy": + return get_version_from_init_py( + airflow_root / "dev" / "mypy" / "src" / "airflow_mypy" / "__init__.py" + ) + if package_name == "task-sdk": return get_version_from_init_py(airflow_root / "task-sdk" / "src" / "airflow" / "sdk" / "__init__.py") diff --git a/scripts/ci/prek/mypy_folder.py b/scripts/ci/prek/mypy_folder.py index 420017c161616..860755ff6894e 100755 --- a/scripts/ci/prek/mypy_folder.py +++ b/scripts/ci/prek/mypy_folder.py @@ -45,6 +45,7 @@ "airflow-core", *[f"providers/{provider_id.replace('.', '/')}" for provider_id in get_all_provider_ids()], "dev", + "dev/mypy", "scripts", "devel-common", "task-sdk", diff --git a/scripts/tests/ci/docs/test_store_stable_versions.py b/scripts/tests/ci/docs/test_store_stable_versions.py index ef02e2e5787d2..9755256dbbc1f 100644 --- a/scripts/tests/ci/docs/test_store_stable_versions.py +++ b/scripts/tests/ci/docs/test_store_stable_versions.py @@ -155,6 +155,13 @@ def test_apache_airflow_ctl(self, tmp_path): assert get_package_version("apache-airflow-ctl", tmp_path) == "0.1.3" + def test_apache_airflow_mypy(self, tmp_path): + init_file = tmp_path / "dev" / "mypy" / "src" / "airflow_mypy" / "__init__.py" + init_file.parent.mkdir(parents=True) + init_file.write_text('__version__ = "0.2.0"\n') + + assert get_package_version("apache-airflow-mypy", tmp_path) == "0.2.0" + def test_task_sdk(self, tmp_path): init_file = tmp_path / "task-sdk" / "src" / "airflow" / "sdk" / "__init__.py" init_file.parent.mkdir(parents=True) @@ -212,6 +219,9 @@ def _create_fake_airflow_root(self, tmp_path): ctl_init = airflow_root / "airflow-ctl" / "src" / "airflowctl" / "__init__.py" ctl_init.parent.mkdir(parents=True) ctl_init.write_text('__version__ = "0.1.3"\n') + mypy_init = airflow_root / "dev" / "mypy" / "src" / "airflow_mypy" / "__init__.py" + mypy_init.parent.mkdir(parents=True) + mypy_init.write_text('__version__ = "0.2.0"\n') return airflow_root def _create_docs_build_dir(self, tmp_path, packages): @@ -225,7 +235,9 @@ def _create_docs_build_dir(self, tmp_path, packages): def test_creates_stable_txt(self, tmp_path, monkeypatch): airflow_root = self._create_fake_airflow_root(tmp_path) - docs_dir = self._create_docs_build_dir(tmp_path, ["apache-airflow", "apache-airflow-ctl"]) + docs_dir = self._create_docs_build_dir( + tmp_path, ["apache-airflow", "apache-airflow-ctl", "apache-airflow-mypy"] + ) monkeypatch.setenv("DOCS_BUILD_DIR", str(docs_dir)) monkeypatch.setenv("AIRFLOW_ROOT", str(airflow_root)) @@ -235,6 +247,7 @@ def test_creates_stable_txt(self, tmp_path, monkeypatch): assert result == 0 assert (docs_dir / "apache-airflow" / "stable.txt").read_text() == "3.2.0\n" assert (docs_dir / "apache-airflow-ctl" / "stable.txt").read_text() == "0.1.3\n" + assert (docs_dir / "apache-airflow-mypy" / "stable.txt").read_text() == "0.2.0\n" def test_creates_versioned_directory(self, tmp_path, monkeypatch): airflow_root = self._create_fake_airflow_root(tmp_path) diff --git a/uv.lock b/uv.lock index ef8a6032697c5..fd05489f13a90 100644 --- a/uv.lock +++ b/uv.lock @@ -2513,6 +2513,7 @@ docs = [ { name = "sphinxcontrib-serializinghtml" }, { name = "sphinxcontrib-spelling" }, { name = "swagger-plugin-for-sphinx" }, + { name = "tabulate" }, ] docs-gen = [ { name = "diagrams" }, @@ -2702,6 +2703,7 @@ requires-dist = [ { name = "sqlalchemy", extras = ["asyncio"], marker = "extra == 'sqlalchemy'", specifier = ">=1.4.49" }, { name = "sqlalchemy-utils", marker = "extra == 'sqlalchemy'", specifier = ">=0.41.2" }, { name = "swagger-plugin-for-sphinx", marker = "extra == 'docs'", specifier = ">=7.1.0,!=7.2.0" }, + { name = "tabulate", marker = "extra == 'docs'", specifier = ">=0.9.0" }, { name = "time-machine", extras = ["dateutil"], specifier = ">=3.0.0" }, { name = "towncrier", marker = "extra == 'devscripts'", specifier = ">=23.11.0" }, { name = "twine", marker = "extra == 'devscripts'", specifier = ">=4.0.2" }, @@ -2876,9 +2878,21 @@ dependencies = [ { name = "mypy" }, ] +[package.dev-dependencies] +docs = [ + { name = "apache-airflow-core" }, + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + [package.metadata] requires-dist = [{ name = "mypy", specifier = ">=1.0.0" }] +[package.metadata.requires-dev] +docs = [ + { name = "apache-airflow-core", editable = "airflow-core" }, + { name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }, +] + [[package]] name = "apache-airflow-providers" version = "0.0.1"