Make pytest treat warnings as errors (#335)

Warnings are usually a sign of something that can be improved or even
wrong, like tasks not being properly awaited or cancelled, so it's a
good idea to treat them as errors to avoid them getting unnoticed.

This PR also moves `pytest` options to the `pyproject.toml` files, so
they are easier to change and more flexible.

Author: llucax

RELEASE_NOTES.md

@@ -6,19 +6,26 @@
 
 ## Upgrading
 
-<!-- Here goes notes on how to upgrade from previous versions, including deprecations and what they should be replaced with -->
+- The `nox` default `pytest` session doesn't pass `-W=all -vv` to `pytest` anymore. You can use the `pyproject.toml` file to configure default options for `pytest`, for example:
+
+    ```toml
+    [tool.pytest.ini_options]
+    addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv"
+    ```
 
 ### Cookiecutter template
 
-<!-- Here upgrade steps for cookiecutter specifically -->
+All upgrading should be done via the migration script or regenerating the templates. But you might still need to adapt your code:
+
+- `pytest` now uses `-Werror` by default (but still treat deprecations as normal warnings), so if your tests run with warnings, they will now be turned to errors, and you'll need to fix them.
 
 ## New Features
 
 <!-- Here goes the main new features and examples or instructions on how to use them -->
 
 ### Cookiecutter template
 
-<!-- Here new features for cookiecutter specifically -->
+- `pytest` now uses `-Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning` by default. Deprecations are still treated as warnings, as when testing with the `pytest_min` session is normal to get deprecation warnings as we are using old versions of dependencies.
 
 ## Bug Fixes
 

cookiecutter/migrate.py

@@ -29,10 +29,41 @@
 
 def main() -> None:
     """Run the migration steps."""
+    add_default_pytest_options()
+
     # Add a separation line like this one after each migration step.
     print("=" * 72)
 
 
+def add_default_pytest_options() -> None:
+    """Add default pytest options to pyproject.toml."""
+    pyproject_toml = Path("pyproject.toml")
+    pyproject_toml_content = pyproject_toml.read_text(encoding="utf-8")
+    marker = "[tool.pytest.ini_options]\n"
+    new_options = (
+        "-W=all Werror -Wdefault::DeprecationWarning "
+        "-Wdefault::PendingDeprecationWarning -vv"
+    )
+
+    print(f"Adding default pytest options to {pyproject_toml}...")
+    if pyproject_toml_content.find(marker) == -1:
+        print(
+            "Couldn't find the the {marker.strip()} marker in pyproject.toml, skipping update."
+        )
+        return
+
+    if pyproject_toml_content.find("\naddopts") >= 0:
+        print("It looks like some options are already configured, skipping update.")
+        manual_step(f"Please consider `{new_options}` if they are not there yet.")
+        return
+
+    replace_file_contents_atomically(
+        pyproject_toml,
+        marker,
+        marker + f'addopts = "{new_options}"\n',
+    )
+
+
 def apply_patch(patch_content: str) -> None:
     """Apply a patch using the patch utility."""
     subprocess.run(["patch", "-p1"], input=patch_content.encode(), check=True)

cookiecutter/{{cookiecutter.github_repo_name}}/pyproject.toml

@@ -203,6 +203,7 @@ disable = [
 
 [tool.pytest.ini_options]
 {%- if cookiecutter.type != "api" %}
+addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv"
 testpaths = ["tests", "src"]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"

pyproject.toml

@@ -205,6 +205,7 @@ module = [
 ignore_missing_imports = true
 
 [tool.pytest.ini_options]
+addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv"
 testpaths = ["src", "tests"]
 markers = [
   "integration: integration tests (deselect with '-m \"not integration\"')",

src/frequenz/repo/config/nox/default.py

@@ -36,10 +36,7 @@
         "--check",
     ],
     mypy=[],
-    pytest=[
-        "-W=all",
-        "-vv",
-    ],
+    pytest=[],
 )
 """Default command-line options for all types of repositories."""
 

tests_golden/integration/test_cookiecutter_generation/actor/frequenz-actor-test/pyproject.toml

@@ -153,6 +153,7 @@ disable = [
 ]
 
 [tool.pytest.ini_options]
+addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv"
 testpaths = ["tests", "src"]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"

tests_golden/integration/test_cookiecutter_generation/app/frequenz-app-test/pyproject.toml

@@ -152,6 +152,7 @@ disable = [
 ]
 
 [tool.pytest.ini_options]
+addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv"
 testpaths = ["tests", "src"]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"

tests_golden/integration/test_cookiecutter_generation/lib/frequenz-test-python/pyproject.toml

@@ -149,6 +149,7 @@ disable = [
 ]
 
 [tool.pytest.ini_options]
+addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv"
 testpaths = ["tests", "src"]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"

tests_golden/integration/test_cookiecutter_generation/model/frequenz-model-test/pyproject.toml

@@ -153,6 +153,7 @@ disable = [
 ]
 
 [tool.pytest.ini_options]
+addopts = "-W=all -Werror -Wdefault::DeprecationWarning -Wdefault::PendingDeprecationWarning -vv"
 testpaths = ["tests", "src"]
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "function"