From 752a499787ffae62c75d9e66a3b06add79644ada Mon Sep 17 00:00:00 2001 From: robot-contrib Date: Tue, 25 Feb 2025 12:32:58 +0300 Subject: [PATCH] Update contrib/python/black to 25.1.0 commit_hash:1d868f3afe3ed85f1cb0cf7e155f28fa33f81f19 --- contrib/python/black/.dist-info/METADATA | 63 +++++- contrib/python/black/README.md | 4 +- contrib/python/black/_black_version.py | 2 +- contrib/python/black/black/__init__.py | 26 +-- contrib/python/black/black/brackets.py | 3 +- contrib/python/black/black/cache.py | 3 +- contrib/python/black/black/comments.py | 11 +- contrib/python/black/black/concurrency.py | 3 +- contrib/python/black/black/debug.py | 3 +- contrib/python/black/black/files.py | 13 +- .../python/black/black/handle_ipynb_magics.py | 31 ++- contrib/python/black/black/linegen.py | 180 ++++++++++-------- contrib/python/black/black/lines.py | 16 +- contrib/python/black/black/mode.py | 12 +- contrib/python/black/black/nodes.py | 40 ++-- contrib/python/black/black/parsing.py | 2 +- contrib/python/black/black/ranges.py | 3 +- .../black/black/resources/black.schema.json | 10 +- contrib/python/black/black/strings.py | 11 +- contrib/python/black/black/trans.py | 95 ++++----- contrib/python/black/blackd/__init__.py | 10 +- contrib/python/black/blackd/middlewares.py | 2 +- contrib/python/black/blib2to3/pgen2/driver.py | 3 +- .../python/black/blib2to3/pgen2/literals.py | 3 +- contrib/python/black/blib2to3/pgen2/parse.py | 3 +- contrib/python/black/blib2to3/pgen2/pgen.py | 7 +- .../python/black/blib2to3/pgen2/tokenize.py | 8 +- contrib/python/black/blib2to3/pytree.py | 3 +- 28 files changed, 329 insertions(+), 241 deletions(-) diff --git a/contrib/python/black/.dist-info/METADATA b/contrib/python/black/.dist-info/METADATA index 9959ac2a9ce..64273a7e12c 100644 --- a/contrib/python/black/.dist-info/METADATA +++ b/contrib/python/black/.dist-info/METADATA @@ -1,13 +1,13 @@ -Metadata-Version: 2.3 +Metadata-Version: 2.4 Name: black -Version: 24.10.0 +Version: 25.1.0 Summary: The uncompromising code formatter. Project-URL: Documentation, https://black.readthedocs.io/ Project-URL: Changelog, https://github.com/psf/black/blob/main/CHANGES.md Project-URL: Repository, https://github.com/psf/black Project-URL: Issues, https://github.com/psf/black/issues Author-email: Łukasz Langa -License: MIT +License-Expression: MIT License-File: AUTHORS.md License-File: LICENSE Keywords: automation,autopep8,formatter,gofmt,pyfmt,rustfmt,yapf @@ -84,7 +84,7 @@ Try it out now using the [Black Playground](https://black.vercel.app). Watch the ### Installation -_Black_ can be installed by running `pip install black`. It requires Python 3.8+ to run. +_Black_ can be installed by running `pip install black`. It requires Python 3.9+ to run. If you want to format Jupyter Notebooks, install with `pip install "black[jupyter]"`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: @@ -183,7 +183,7 @@ SQLAlchemy, Poetry, PyPA applications (Warehouse, Bandersnatch, Pipenv, virtuale pandas, Pillow, Twisted, LocalStack, every Datadog Agent Integration, Home Assistant, Zulip, Kedro, OpenOA, FLORIS, ORBIT, WOMBAT, and many more. -The following organizations use _Black_: Facebook, Dropbox, KeepTruckin, Lyft, Mozilla, +The following organizations use _Black_: Dropbox, KeepTruckin, Lyft, Mozilla, Quora, Duolingo, QuantumBlack, Tesla, Archer Aviation. Are we missing anyone? Let us know. @@ -276,6 +276,59 @@ Flying Circus is expected. We are not savages. And if you _really_ need to slap somebody, do it with a fish while dancing. # Change Log +## 25.1.0 + +### Highlights + +This release introduces the new 2025 stable style (#4558), stabilizing +the following changes: + +- Normalize casing of Unicode escape characters in strings to lowercase (#2916) +- Fix inconsistencies in whether certain strings are detected as docstrings (#4095) +- Consistently add trailing commas to typed function parameters (#4164) +- Remove redundant parentheses in if guards for case blocks (#4214) +- Add parentheses to if clauses in case blocks when the line is too long (#4269) +- Whitespace before `# fmt: skip` comments is no longer normalized (#4146) +- Fix line length computation for certain expressions that involve the power operator (#4154) +- Check if there is a newline before the terminating quotes of a docstring (#4185) +- Fix type annotation spacing between `*` and more complex type variable tuple (#4440) + +The following changes were not in any previous release: + +- Remove parentheses around sole list items (#4312) +- Generic function definitions are now formatted more elegantly: parameters are + split over multiple lines first instead of type parameter definitions (#4553) + +### Stable style + +- Fix formatting cells in IPython notebooks with magic methods and starting or trailing + empty lines (#4484) +- Fix crash when formatting `with` statements containing tuple generators/unpacking + (#4538) + +### Preview style + +- Fix/remove string merging changing f-string quotes on f-strings with internal quotes + (#4498) +- Collapse multiple empty lines after an import into one (#4489) +- Prevent `string_processing` and `wrap_long_dict_values_in_parens` from removing + parentheses around long dictionary values (#4377) +- Move `wrap_long_dict_values_in_parens` from the unstable to preview style (#4561) + +### Packaging + +- Store license identifier inside the `License-Expression` metadata field, see + [PEP 639](https://peps.python.org/pep-0639/). (#4479) + +### Performance + +- Speed up the `is_fstring_start` function in Black's tokenizer (#4541) + +### Integrations + +- If using stdin with `--stdin-filename` set to a force excluded path, stdin won't be + formatted. (#4539) + ## 24.10.0 ### Highlights diff --git a/contrib/python/black/README.md b/contrib/python/black/README.md index fb8170b626a..cb3cf71f3dc 100644 --- a/contrib/python/black/README.md +++ b/contrib/python/black/README.md @@ -38,7 +38,7 @@ Try it out now using the [Black Playground](https://black.vercel.app). Watch the ### Installation -_Black_ can be installed by running `pip install black`. It requires Python 3.8+ to run. +_Black_ can be installed by running `pip install black`. It requires Python 3.9+ to run. If you want to format Jupyter Notebooks, install with `pip install "black[jupyter]"`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: @@ -137,7 +137,7 @@ SQLAlchemy, Poetry, PyPA applications (Warehouse, Bandersnatch, Pipenv, virtuale pandas, Pillow, Twisted, LocalStack, every Datadog Agent Integration, Home Assistant, Zulip, Kedro, OpenOA, FLORIS, ORBIT, WOMBAT, and many more. -The following organizations use _Black_: Facebook, Dropbox, KeepTruckin, Lyft, Mozilla, +The following organizations use _Black_: Dropbox, KeepTruckin, Lyft, Mozilla, Quora, Duolingo, QuantumBlack, Tesla, Archer Aviation. Are we missing anyone? Let us know. diff --git a/contrib/python/black/_black_version.py b/contrib/python/black/_black_version.py index 33a0accd594..bc98431b02b 100644 --- a/contrib/python/black/_black_version.py +++ b/contrib/python/black/_black_version.py @@ -1 +1 @@ -version = "24.10.0" +version = "25.1.0" diff --git a/contrib/python/black/black/__init__.py b/contrib/python/black/black/__init__.py index a94f7fc29a0..93a08a8d88a 100644 --- a/contrib/python/black/black/__init__.py +++ b/contrib/python/black/black/__init__.py @@ -5,24 +5,22 @@ import sys import tokenize import traceback -from contextlib import contextmanager -from dataclasses import replace -from datetime import datetime, timezone -from enum import Enum -from json.decoder import JSONDecodeError -from pathlib import Path -from typing import ( - Any, +from collections.abc import ( Collection, Generator, Iterator, MutableMapping, - Optional, - Pattern, Sequence, Sized, - Union, ) +from contextlib import contextmanager +from dataclasses import replace +from datetime import datetime, timezone +from enum import Enum +from json.decoder import JSONDecodeError +from pathlib import Path +from re import Pattern +from typing import Any, Optional, Union import click from click.core import ParameterSource @@ -751,6 +749,12 @@ def get_sources( for s in src: if s == "-" and stdin_filename: path = Path(stdin_filename) + if path_is_excluded(stdin_filename, force_exclude): + report.path_ignored( + path, + "--stdin-filename matches the --force-exclude regular expression", + ) + continue is_stdin = True else: path = Path(s) diff --git a/contrib/python/black/black/brackets.py b/contrib/python/black/black/brackets.py index 4a994a9d5c7..c2e8be4348e 100644 --- a/contrib/python/black/black/brackets.py +++ b/contrib/python/black/black/brackets.py @@ -1,7 +1,8 @@ """Builds on top of nodes.py to track brackets.""" +from collections.abc import Iterable, Sequence from dataclasses import dataclass, field -from typing import Final, Iterable, Optional, Sequence, Union +from typing import Final, Optional, Union from black.nodes import ( BRACKET, diff --git a/contrib/python/black/black/cache.py b/contrib/python/black/black/cache.py index 8811a79d79c..ef9d99a7b90 100644 --- a/contrib/python/black/black/cache.py +++ b/contrib/python/black/black/cache.py @@ -5,9 +5,10 @@ import pickle import sys import tempfile +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path -from typing import Iterable, NamedTuple +from typing import NamedTuple from platformdirs import user_cache_dir diff --git a/contrib/python/black/black/comments.py b/contrib/python/black/black/comments.py index cd37c440290..f42a51033db 100644 --- a/contrib/python/black/black/comments.py +++ b/contrib/python/black/black/comments.py @@ -1,9 +1,10 @@ import re +from collections.abc import Collection, Iterator from dataclasses import dataclass from functools import lru_cache -from typing import Collection, Final, Iterator, Optional, Union +from typing import Final, Optional, Union -from black.mode import Mode, Preview +from black.mode import Mode from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -234,11 +235,7 @@ def convert_one_fmt_off_pair( standalone_comment_prefix += fmt_off_prefix hidden_value = comment.value + "\n" + hidden_value if is_fmt_skip: - hidden_value += ( - comment.leading_whitespace - if Preview.no_normalize_fmt_skip_whitespace in mode - else " " - ) + comment.value + hidden_value += comment.leading_whitespace + comment.value if hidden_value.endswith("\n"): # That happens when one of the `ignored_nodes` ended with a NEWLINE # leaf (possibly followed by a DEDENT). diff --git a/contrib/python/black/black/concurrency.py b/contrib/python/black/black/concurrency.py index 8079100f8f7..4b3cf48d901 100644 --- a/contrib/python/black/black/concurrency.py +++ b/contrib/python/black/black/concurrency.py @@ -10,10 +10,11 @@ import signal import sys import traceback +from collections.abc import Iterable from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor from multiprocessing import Manager from pathlib import Path -from typing import Any, Iterable, Optional +from typing import Any, Optional from mypy_extensions import mypyc_attr diff --git a/contrib/python/black/black/debug.py b/contrib/python/black/black/debug.py index 34a9f32e5cb..939b20eee5e 100644 --- a/contrib/python/black/black/debug.py +++ b/contrib/python/black/black/debug.py @@ -1,5 +1,6 @@ +from collections.abc import Iterator from dataclasses import dataclass, field -from typing import Any, Iterator, TypeVar, Union +from typing import Any, TypeVar, Union from black.nodes import Visitor from black.output import out diff --git a/contrib/python/black/black/files.py b/contrib/python/black/black/files.py index 82da47919c7..72c5eddf9c0 100644 --- a/contrib/python/black/black/files.py +++ b/contrib/python/black/black/files.py @@ -1,18 +1,11 @@ import io import os import sys +from collections.abc import Iterable, Iterator, Sequence from functools import lru_cache from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Iterable, - Iterator, - Optional, - Pattern, - Sequence, - Union, -) +from re import Pattern +from typing import TYPE_CHECKING, Any, Optional, Union from mypy_extensions import mypyc_attr from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet diff --git a/contrib/python/black/black/handle_ipynb_magics.py b/contrib/python/black/black/handle_ipynb_magics.py index 792d22595aa..dd680bffffb 100644 --- a/contrib/python/black/black/handle_ipynb_magics.py +++ b/contrib/python/black/black/handle_ipynb_magics.py @@ -43,7 +43,6 @@ "time", "timeit", )) -TOKEN_HEX = secrets.token_hex @dataclasses.dataclass(frozen=True) @@ -160,7 +159,7 @@ def mask_cell(src: str) -> tuple[str, list[Replacement]]: becomes - "25716f358c32750e" + b"25716f358c32750" 'foo' The replacements are returned, along with the transformed code. @@ -178,18 +177,32 @@ def mask_cell(src: str) -> tuple[str, list[Replacement]]: from IPython.core.inputtransformer2 import TransformerManager transformer_manager = TransformerManager() + # A side effect of the following transformation is that it also removes any + # empty lines at the beginning of the cell. transformed = transformer_manager.transform_cell(src) transformed, cell_magic_replacements = replace_cell_magics(transformed) replacements += cell_magic_replacements transformed = transformer_manager.transform_cell(transformed) transformed, magic_replacements = replace_magics(transformed) - if len(transformed.splitlines()) != len(src.splitlines()): + if len(transformed.strip().splitlines()) != len(src.strip().splitlines()): # Multi-line magic, not supported. raise NothingChanged replacements += magic_replacements return transformed, replacements +def create_token(n_chars: int) -> str: + """Create a randomly generated token that is n_chars characters long.""" + assert n_chars > 0 + n_bytes = max(n_chars // 2 - 1, 1) + token = secrets.token_hex(n_bytes) + if len(token) + 3 > n_chars: + token = token[:-1] + # We use a bytestring so that the string does not get interpreted + # as a docstring. + return f'b"{token}"' + + def get_token(src: str, magic: str) -> str: """Return randomly generated token to mask IPython magic with. @@ -199,11 +212,11 @@ def get_token(src: str, magic: str) -> str: not already present anywhere else in the cell. """ assert magic - nbytes = max(len(magic) // 2 - 1, 1) - token = TOKEN_HEX(nbytes) + n_chars = len(magic) + token = create_token(n_chars) counter = 0 while token in src: - token = TOKEN_HEX(nbytes) + token = create_token(n_chars) counter += 1 if counter > 100: raise AssertionError( @@ -211,9 +224,7 @@ def get_token(src: str, magic: str) -> str: "Please report a bug on https://github.com/psf/black/issues. " f"The magic might be helpful: {magic}" ) from None - if len(token) + 2 < len(magic): - token = f"{token}." - return f'"{token}"' + return token def replace_cell_magics(src: str) -> tuple[str, list[Replacement]]: @@ -269,7 +280,7 @@ def replace_magics(src: str) -> tuple[str, list[Replacement]]: magic_finder = MagicFinder() magic_finder.visit(ast.parse(src)) new_srcs = [] - for i, line in enumerate(src.splitlines(), start=1): + for i, line in enumerate(src.split("\n"), start=1): if i in magic_finder.magics: offsets_and_magics = magic_finder.magics[i] if len(offsets_and_magics) != 1: # pragma: nocover diff --git a/contrib/python/black/black/linegen.py b/contrib/python/black/black/linegen.py index 107fa69d052..ee65a7a6e40 100644 --- a/contrib/python/black/black/linegen.py +++ b/contrib/python/black/black/linegen.py @@ -4,10 +4,11 @@ import re import sys +from collections.abc import Collection, Iterator from dataclasses import replace from enum import Enum, auto from functools import partial, wraps -from typing import Collection, Iterator, Optional, Union, cast +from typing import Optional, Union, cast from black.brackets import ( COMMA_PRIORITY, @@ -44,6 +45,7 @@ is_atom_with_invisible_parens, is_docstring, is_empty_tuple, + is_generator, is_lpar_token, is_multiline_string, is_name_token, @@ -54,6 +56,7 @@ is_rpar_token, is_stub_body, is_stub_suite, + is_tuple_containing_star, is_tuple_containing_walrus, is_type_ignore_comment_string, is_vararg, @@ -64,7 +67,7 @@ ) from black.numerics import normalize_numeric_literal from black.strings import ( - fix_docstring, + fix_multiline_docstring, get_string_prefix, normalize_string_prefix, normalize_string_quotes, @@ -411,10 +414,9 @@ def foo(a: (int), b: (float) = 7): ... yield from self.visit_default(node) def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: - if Preview.hex_codes_in_unicode_sequences in self.mode: - normalize_unicode_escape_sequences(leaf) + normalize_unicode_escape_sequences(leaf) - if is_docstring(leaf, self.mode) and not re.search(r"\\\s*\n", leaf.value): + if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. if self.mode.string_normalization: @@ -441,7 +443,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: indent = " " * 4 * self.current_line.depth if is_multiline_string(leaf): - docstring = fix_docstring(docstring, indent) + docstring = fix_multiline_docstring(docstring, indent) else: docstring = docstring.strip() @@ -485,10 +487,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: and len(indent) + quote_len <= self.mode.line_length and not has_trailing_backslash ): - if ( - Preview.docstring_check_for_newline in self.mode - and leaf.value[-1 - quote_len] == "\n" - ): + if leaf.value[-1 - quote_len] == "\n": leaf.value = prefix + quote + docstring + quote else: leaf.value = prefix + quote + docstring + "\n" + indent + quote @@ -506,6 +505,19 @@ def visit_NUMBER(self, leaf: Leaf) -> Iterator[Line]: normalize_numeric_literal(leaf) yield from self.visit_default(leaf) + def visit_atom(self, node: Node) -> Iterator[Line]: + """Visit any atom""" + if len(node.children) == 3: + first = node.children[0] + last = node.children[-1] + if (first.type == token.LSQB and last.type == token.RSQB) or ( + first.type == token.LBRACE and last.type == token.RBRACE + ): + # Lists or sets of one item + maybe_make_parens_invisible_in_atom(node.children[1], parent=node) + + yield from self.visit_default(node) + def visit_fstring(self, node: Node) -> Iterator[Line]: # currently we don't want to format and split f-strings at all. string_leaf = fstring_to_string(node) @@ -583,8 +595,7 @@ def __post_init__(self) -> None: # PEP 634 self.visit_match_stmt = self.visit_match_case self.visit_case_block = self.visit_match_case - if Preview.remove_redundant_guard_parens in self.mode: - self.visit_guard = partial(v, keywords=Ø, parens={"if"}) + self.visit_guard = partial(v, keywords=Ø, parens={"if"}) def _hugging_power_ops_line_to_string( @@ -768,26 +779,29 @@ def left_hand_split( Prefer RHS otherwise. This is why this function is not symmetrical with :func:`right_hand_split` which also handles optional parentheses. """ - tail_leaves: list[Leaf] = [] - body_leaves: list[Leaf] = [] - head_leaves: list[Leaf] = [] - current_leaves = head_leaves - matching_bracket: Optional[Leaf] = None - for leaf in line.leaves: - if ( - current_leaves is body_leaves - and leaf.type in CLOSING_BRACKETS - and leaf.opening_bracket is matching_bracket - and isinstance(matching_bracket, Leaf) - ): - ensure_visible(leaf) - ensure_visible(matching_bracket) - current_leaves = tail_leaves if body_leaves else head_leaves - current_leaves.append(leaf) - if current_leaves is head_leaves: - if leaf.type in OPENING_BRACKETS: - matching_bracket = leaf - current_leaves = body_leaves + for leaf_type in [token.LPAR, token.LSQB]: + tail_leaves: list[Leaf] = [] + body_leaves: list[Leaf] = [] + head_leaves: list[Leaf] = [] + current_leaves = head_leaves + matching_bracket: Optional[Leaf] = None + for leaf in line.leaves: + if ( + current_leaves is body_leaves + and leaf.type in CLOSING_BRACKETS + and leaf.opening_bracket is matching_bracket + and isinstance(matching_bracket, Leaf) + ): + ensure_visible(leaf) + ensure_visible(matching_bracket) + current_leaves = tail_leaves if body_leaves else head_leaves + current_leaves.append(leaf) + if current_leaves is head_leaves: + if leaf.type == leaf_type: + matching_bracket = leaf + current_leaves = body_leaves + if matching_bracket and tail_leaves: + break if not matching_bracket or not tail_leaves: raise CannotSplit("No brackets found") @@ -954,29 +968,7 @@ def _maybe_split_omitting_optional_parens( try: # The RHSResult Omitting Optional Parens. rhs_oop = _first_right_hand_split(line, omit=omit) - is_split_right_after_equal = ( - len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL - ) - rhs_head_contains_brackets = any( - leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1] - ) - # the -1 is for the ending optional paren - rhs_head_short_enough = is_line_short_enough( - rhs.head, mode=replace(mode, line_length=mode.line_length - 1) - ) - rhs_head_explode_blocked_by_magic_trailing_comma = ( - rhs.head.magic_trailing_comma is None - ) - if ( - not ( - is_split_right_after_equal - and rhs_head_contains_brackets - and rhs_head_short_enough - and rhs_head_explode_blocked_by_magic_trailing_comma - ) - # the omit optional parens split is preferred by some other reason - or _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode) - ): + if _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode): yield from _maybe_split_omitting_optional_parens( rhs_oop, line, mode, features=features, omit=omit ) @@ -987,8 +979,15 @@ def _maybe_split_omitting_optional_parens( if line.is_chained_assignment: pass - elif not can_be_split(rhs.body) and not is_line_short_enough( - rhs.body, mode=mode + elif ( + not can_be_split(rhs.body) + and not is_line_short_enough(rhs.body, mode=mode) + and not ( + Preview.wrap_long_dict_values_in_parens + and rhs.opening_bracket.parent + and rhs.opening_bracket.parent.parent + and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker + ) ): raise CannotSplit( "Splitting failed, body is still too long and can't be split." @@ -1019,6 +1018,44 @@ def _prefer_split_rhs_oop_over_rhs( Returns whether we should prefer the result from a split omitting optional parens (rhs_oop) over the original (rhs). """ + # contains unsplittable type ignore + if ( + rhs_oop.head.contains_unsplittable_type_ignore() + or rhs_oop.body.contains_unsplittable_type_ignore() + or rhs_oop.tail.contains_unsplittable_type_ignore() + ): + return True + + # Retain optional parens around dictionary values + if ( + Preview.wrap_long_dict_values_in_parens + and rhs.opening_bracket.parent + and rhs.opening_bracket.parent.parent + and rhs.opening_bracket.parent.parent.type == syms.dictsetmaker + and rhs.body.bracket_tracker.delimiters + ): + # Unless the split is inside the key + return any(leaf.type == token.COLON for leaf in rhs_oop.tail.leaves) + + # the split is right after `=` + if not (len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL): + return True + + # the left side of assignment contains brackets + if not any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]): + return True + + # the left side of assignment is short enough (the -1 is for the ending optional + # paren) + if not is_line_short_enough( + rhs.head, mode=replace(mode, line_length=mode.line_length - 1) + ): + return True + + # the left side of assignment won't explode further because of magic trailing comma + if rhs.head.magic_trailing_comma is not None: + return True + # If we have multiple targets, we prefer more `=`s on the head vs pushing them to # the body rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL) @@ -1046,10 +1083,6 @@ def _prefer_split_rhs_oop_over_rhs( # the first line is short enough and is_line_short_enough(rhs_oop.head, mode=mode) ) - # contains unsplittable type ignore - or rhs_oop.head.contains_unsplittable_type_ignore() - or rhs_oop.body.contains_unsplittable_type_ignore() - or rhs_oop.tail.contains_unsplittable_type_ignore() ) @@ -1094,12 +1127,7 @@ def _ensure_trailing_comma( return False # Don't add commas if we already have any commas if any( - leaf.type == token.COMMA - and ( - Preview.typed_params_trailing_comma not in original.mode - or not is_part_of_annotation(leaf) - ) - for leaf in leaves + leaf.type == token.COMMA and not is_part_of_annotation(leaf) for leaf in leaves ): return False @@ -1380,11 +1408,7 @@ def normalize_invisible_parens( # noqa: C901 ) # Add parentheses around if guards in case blocks - if ( - isinstance(child, Node) - and child.type == syms.guard - and Preview.parens_for_long_if_clauses_in_case_block in mode - ): + if isinstance(child, Node) and child.type == syms.guard: normalize_invisible_parens( child, parens_after={"if"}, mode=mode, features=features ) @@ -1611,6 +1635,8 @@ def maybe_make_parens_invisible_in_atom( and max_delimiter_priority_in_atom(node) >= COMMA_PRIORITY ) or is_tuple_containing_walrus(node) + or is_tuple_containing_star(node) + or is_generator(node) ): return False @@ -1642,9 +1668,6 @@ def maybe_make_parens_invisible_in_atom( not is_type_ignore_comment_string(middle.prefix.strip()) ): first.value = "" - if first.prefix.strip(): - # Preserve comments before first paren - middle.prefix = first.prefix + middle.prefix last.value = "" maybe_make_parens_invisible_in_atom( middle, @@ -1656,6 +1679,13 @@ def maybe_make_parens_invisible_in_atom( # Strip the invisible parens from `middle` by replacing # it with the child in-between the invisible parens middle.replace(middle.children[1]) + + if middle.children[0].prefix.strip(): + # Preserve comments before first paren + middle.children[1].prefix = ( + middle.children[0].prefix + middle.children[1].prefix + ) + if middle.children[-1].prefix.strip(): # Preserve comments before last paren last.prefix = middle.children[-1].prefix + last.prefix diff --git a/contrib/python/black/black/lines.py b/contrib/python/black/black/lines.py index a8c6ef66f68..2a719def3c9 100644 --- a/contrib/python/black/black/lines.py +++ b/contrib/python/black/black/lines.py @@ -1,7 +1,8 @@ import itertools import math +from collections.abc import Callable, Iterator, Sequence from dataclasses import dataclass, field -from typing import Callable, Iterator, Optional, Sequence, TypeVar, Union, cast +from typing import Optional, TypeVar, Union, cast from black.brackets import COMMA_PRIORITY, DOT_PRIORITY, BracketTracker from black.mode import Mode, Preview @@ -203,9 +204,7 @@ def _is_triple_quoted_string(self) -> bool: @property def is_docstring(self) -> bool: """Is the line a docstring?""" - if Preview.unify_docstring_detection not in self.mode: - return self._is_triple_quoted_string - return bool(self) and is_docstring(self.leaves[0], self.mode) + return bool(self) and is_docstring(self.leaves[0]) @property def is_chained_assignment(self) -> bool: @@ -670,6 +669,15 @@ def _maybe_empty_lines(self, current_line: Line) -> tuple[int, int]: # noqa: C9 current_line, before, user_had_newline ) + if ( + self.previous_line.is_import + and self.previous_line.depth == 0 + and current_line.depth == 0 + and not current_line.is_import + and Preview.always_one_newline_after_import in self.mode + ): + return 1, 0 + if ( self.previous_line.is_import and not current_line.is_import diff --git a/contrib/python/black/black/mode.py b/contrib/python/black/black/mode.py index 02fe1de24db..7335bd12078 100644 --- a/contrib/python/black/black/mode.py +++ b/contrib/python/black/black/mode.py @@ -196,28 +196,18 @@ def supports_feature(target_versions: set[TargetVersion], feature: Feature) -> b class Preview(Enum): """Individual preview style features.""" - hex_codes_in_unicode_sequences = auto() # NOTE: string_processing requires wrap_long_dict_values_in_parens # for https://github.com/psf/black/issues/3117 to be fixed. string_processing = auto() hug_parens_with_braces_and_square_brackets = auto() - unify_docstring_detection = auto() - no_normalize_fmt_skip_whitespace = auto() wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() - typed_params_trailing_comma = auto() - is_simple_lookup_for_doublestar_expression = auto() - docstring_check_for_newline = auto() - remove_redundant_guard_parens = auto() - parens_for_long_if_clauses_in_case_block = auto() - pep646_typed_star_arg_type_var_tuple = auto() + always_one_newline_after_import = auto() UNSTABLE_FEATURES: set[Preview] = { # Many issues, see summary in https://github.com/psf/black/issues/4042 Preview.string_processing, - # See issues #3452 and #4158 - Preview.wrap_long_dict_values_in_parens, # See issue #4159 Preview.multiline_string_handling, # See issue #4036 (crash), #4098, #4099 (proposed tweaks) diff --git a/contrib/python/black/black/nodes.py b/contrib/python/black/black/nodes.py index 470dc248488..3b74e2db0be 100644 --- a/contrib/python/black/black/nodes.py +++ b/contrib/python/black/black/nodes.py @@ -3,7 +3,8 @@ """ import sys -from typing import Final, Generic, Iterator, Literal, Optional, TypeVar, Union +from collections.abc import Iterator +from typing import Final, Generic, Literal, Optional, TypeVar, Union if sys.version_info >= (3, 10): from typing import TypeGuard @@ -13,7 +14,7 @@ from mypy_extensions import mypyc_attr from black.cache import CACHE_DIR -from black.mode import Mode, Preview +from black.mode import Mode from black.strings import get_string_prefix, has_triple_quotes from blib2to3 import pygram from blib2to3.pgen2 import token @@ -243,13 +244,7 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # no elif ( prevp.type == token.STAR and parent_type(prevp) == syms.star_expr - and ( - parent_type(prevp.parent) == syms.subscriptlist - or ( - Preview.pep646_typed_star_arg_type_var_tuple in mode - and parent_type(prevp.parent) == syms.tname_star - ) - ) + and parent_type(prevp.parent) in (syms.subscriptlist, syms.tname_star) ): # No space between typevar tuples or unpacking them. return NO @@ -550,7 +545,7 @@ def is_arith_like(node: LN) -> bool: } -def is_docstring(node: NL, mode: Mode) -> bool: +def is_docstring(node: NL) -> bool: if isinstance(node, Leaf): if node.type != token.STRING: return False @@ -560,8 +555,7 @@ def is_docstring(node: NL, mode: Mode) -> bool: return False if ( - Preview.unify_docstring_detection in mode - and node.parent + node.parent and node.parent.type == syms.simple_stmt and not node.parent.prev_sibling and node.parent.parent @@ -620,6 +614,28 @@ def is_tuple_containing_walrus(node: LN) -> bool: return any(child.type == syms.namedexpr_test for child in gexp.children) +def is_tuple_containing_star(node: LN) -> bool: + """Return True if `node` holds a tuple that contains a star operator.""" + if node.type != syms.atom: + return False + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return any(child.type == syms.star_expr for child in gexp.children) + + +def is_generator(node: LN) -> bool: + """Return True if `node` holds a generator.""" + if node.type != syms.atom: + return False + gexp = unwrap_singleton_parenthesis(node) + if gexp is None or gexp.type != syms.testlist_gexp: + return False + + return any(child.type == syms.old_comp_for for child in gexp.children) + + def is_one_sequence_between( opening: Leaf, closing: Leaf, diff --git a/contrib/python/black/black/parsing.py b/contrib/python/black/black/parsing.py index e139963183a..0019b0c006a 100644 --- a/contrib/python/black/black/parsing.py +++ b/contrib/python/black/black/parsing.py @@ -5,7 +5,7 @@ import ast import sys import warnings -from typing import Collection, Iterator +from collections.abc import Collection, Iterator from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature from black.nodes import syms diff --git a/contrib/python/black/black/ranges.py b/contrib/python/black/black/ranges.py index f8b09a67a01..90649137d2e 100644 --- a/contrib/python/black/black/ranges.py +++ b/contrib/python/black/black/ranges.py @@ -1,8 +1,9 @@ """Functions related to Black's formatting by line ranges feature.""" import difflib +from collections.abc import Collection, Iterator, Sequence from dataclasses import dataclass -from typing import Collection, Iterator, Sequence, Union +from typing import Union from black.nodes import ( LN, diff --git a/contrib/python/black/black/resources/black.schema.json b/contrib/python/black/black/resources/black.schema.json index a536d543fed..b9b61489136 100644 --- a/contrib/python/black/black/resources/black.schema.json +++ b/contrib/python/black/black/resources/black.schema.json @@ -79,19 +79,11 @@ "type": "array", "items": { "enum": [ - "hex_codes_in_unicode_sequences", "string_processing", "hug_parens_with_braces_and_square_brackets", - "unify_docstring_detection", - "no_normalize_fmt_skip_whitespace", "wrap_long_dict_values_in_parens", "multiline_string_handling", - "typed_params_trailing_comma", - "is_simple_lookup_for_doublestar_expression", - "docstring_check_for_newline", - "remove_redundant_guard_parens", - "parens_for_long_if_clauses_in_case_block", - "pep646_typed_star_arg_type_var_tuple" + "always_one_newline_after_import" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/contrib/python/black/black/strings.py b/contrib/python/black/black/strings.py index 0973907bd3c..a3018990ee8 100644 --- a/contrib/python/black/black/strings.py +++ b/contrib/python/black/black/strings.py @@ -5,7 +5,8 @@ import re import sys from functools import lru_cache -from typing import Final, Match, Pattern +from re import Match, Pattern +from typing import Final from black._width_table import WIDTH_TABLE from blib2to3.pytree import Leaf @@ -62,10 +63,9 @@ def lines_with_leading_tabs_expanded(s: str) -> list[str]: return lines -def fix_docstring(docstring: str, prefix: str) -> str: +def fix_multiline_docstring(docstring: str, prefix: str) -> str: # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation - if not docstring: - return "" + assert docstring, "INTERNAL ERROR: Multiline docstrings cannot be empty" lines = lines_with_leading_tabs_expanded(docstring) # Determine minimum indentation (first line doesn't count): indent = sys.maxsize @@ -185,8 +185,7 @@ def normalize_string_quotes(s: str) -> str: orig_quote = "'" new_quote = '"' first_quote_pos = s.find(orig_quote) - if first_quote_pos == -1: - return s # There's an internal error + assert first_quote_pos != -1, f"INTERNAL ERROR: Malformed string {s!r}" prefix = s[:first_quote_pos] unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}") diff --git a/contrib/python/black/black/trans.py b/contrib/python/black/black/trans.py index b44e3cdf0e7..fabc7051108 100644 --- a/contrib/python/black/black/trans.py +++ b/contrib/python/black/black/trans.py @@ -5,27 +5,15 @@ import re from abc import ABC, abstractmethod from collections import defaultdict +from collections.abc import Callable, Collection, Iterable, Iterator, Sequence from dataclasses import dataclass -from typing import ( - Any, - Callable, - ClassVar, - Collection, - Final, - Iterable, - Iterator, - Literal, - Optional, - Sequence, - TypeVar, - Union, -) +from typing import Any, ClassVar, Final, Literal, Optional, TypeVar, Union from mypy_extensions import trait from black.comments import contains_pragma_comment from black.lines import Line, append_leaves -from black.mode import Feature, Mode, Preview +from black.mode import Feature, Mode from black.nodes import ( CLOSING_BRACKETS, OPENING_BRACKETS, @@ -94,18 +82,12 @@ def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool: # Brackets and parentheses indicate calls, subscripts, etc. ... # basically stuff that doesn't count as "simple". Only a NAME lookup # or dotted lookup (eg. NAME.NAME) is OK. - if Preview.is_simple_lookup_for_doublestar_expression not in mode: - return original_is_simple_lookup_func(line, index, kind) - + if kind == -1: + return handle_is_simple_look_up_prev(line, index, {token.RPAR, token.RSQB}) else: - if kind == -1: - return handle_is_simple_look_up_prev( - line, index, {token.RPAR, token.RSQB} - ) - else: - return handle_is_simple_lookup_forward( - line, index, {token.LPAR, token.LSQB} - ) + return handle_is_simple_lookup_forward( + line, index, {token.LPAR, token.LSQB} + ) def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool: # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple @@ -151,30 +133,6 @@ def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool: yield new_line -def original_is_simple_lookup_func( - line: Line, index: int, step: Literal[1, -1] -) -> bool: - if step == -1: - disallowed = {token.RPAR, token.RSQB} - else: - disallowed = {token.LPAR, token.LSQB} - - while 0 <= index < len(line.leaves): - current = line.leaves[index] - if current.type in disallowed: - return False - if current.type not in {token.NAME, token.DOT} or current.value == "for": - # If the current token isn't disallowed, we'll assume this is - # simple as only the disallowed tokens are semantically - # attached to this lookup expression we're checking. Also, - # stop early if we hit the 'for' bit of a comprehension. - return True - - index += step - - return True - - def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: set[int]) -> bool: """ Handling the determination of is_simple_lookup for the lines prior to the doublestar @@ -672,10 +630,10 @@ def make_naked(string: str, string_prefix: str) -> str: """ assert_is_leaf_string(string) if "f" in string_prefix: - f_expressions = ( + f_expressions = [ string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces for span in iter_fexpr_spans(string) - ) + ] debug_expressions_contain_visible_quotes = any( re.search(r".*[\'\"].*(? TResult[None]: - The set of all string prefixes in the string group is of length greater than one and is not equal to {"", "f"}. - The string group consists of raw strings. + - The string group would merge f-strings with different quote types + and internal quotes. - The string group is stringified type annotations. We don't want to process stringified type annotations since pyright doesn't support them spanning multiple string values. (NOTE: mypy, pytype, pyre do @@ -832,6 +792,8 @@ def _validate_msg(line: Line, string_idx: int) -> TResult[None]: i += inc + QUOTE = line.leaves[string_idx].value[-1] + num_of_inline_string_comments = 0 set_of_prefixes = set() num_of_strings = 0 @@ -854,6 +816,19 @@ def _validate_msg(line: Line, string_idx: int) -> TResult[None]: set_of_prefixes.add(prefix) + if ( + "f" in prefix + and leaf.value[-1] != QUOTE + and ( + "'" in leaf.value[len(prefix) + 1 : -1] + or '"' in leaf.value[len(prefix) + 1 : -1] + ) + ): + return TErr( + "StringMerger does NOT merge f-strings with different quote types" + " and internal quotes." + ) + if id(leaf) in line.comments: num_of_inline_string_comments += 1 if contains_pragma_comment(line.comments[id(leaf)]): @@ -882,6 +857,7 @@ class StringParenStripper(StringTransformer): The line contains a string which is surrounded by parentheses and: - The target string is NOT the only argument to a function call. - The target string is NOT a "pointless" string. + - The target string is NOT a dictionary value. - If the target string contains a PERCENT, the brackets are not preceded or followed by an operator with higher precedence than PERCENT. @@ -929,11 +905,14 @@ def do_match(self, line: Line) -> TMatchResult: ): continue - # That LPAR should NOT be preceded by a function name or a closing - # bracket (which could be a function which returns a function or a - # list/dictionary that contains a function)... + # That LPAR should NOT be preceded by a colon (which could be a + # dictionary value), function name, or a closing bracket (which + # could be a function returning a function or a list/dictionary + # containing a function)... if is_valid_index(idx - 2) and ( - LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS + LL[idx - 2].type == token.COLON + or LL[idx - 2].type == token.NAME + or LL[idx - 2].type in CLOSING_BRACKETS ): continue @@ -2259,12 +2238,12 @@ def do_transform( elif right_leaves and right_leaves[-1].type == token.RPAR: # Special case for lambda expressions as dict's value, e.g.: # my_dict = { - # "key": lambda x: f"formatted: {x}, + # "key": lambda x: f"formatted: {x}", # } # After wrapping the dict's value with parentheses, the string is # followed by a RPAR but its opening bracket is lambda's, not # the string's: - # "key": (lambda x: f"formatted: {x}), + # "key": (lambda x: f"formatted: {x}"), opening_bracket = right_leaves[-1].opening_bracket if opening_bracket is not None and opening_bracket in left_leaves: index = left_leaves.index(opening_bracket) diff --git a/contrib/python/black/blackd/__init__.py b/contrib/python/black/blackd/__init__.py index d51b9cec284..86309da0ef0 100644 --- a/contrib/python/black/blackd/__init__.py +++ b/contrib/python/black/blackd/__init__.py @@ -2,7 +2,7 @@ import logging from concurrent.futures import Executor, ProcessPoolExecutor from datetime import datetime, timezone -from functools import partial +from functools import cache, partial from multiprocessing import freeze_support try: @@ -85,12 +85,16 @@ def main(bind_host: str, bind_port: int) -> None: web.run_app(app, host=bind_host, port=bind_port, handle_signals=True, print=None) +@cache +def executor() -> Executor: + return ProcessPoolExecutor() + + def make_app() -> web.Application: app = web.Application( middlewares=[cors(allow_headers=(*BLACK_HEADERS, "Content-Type"))] ) - executor = ProcessPoolExecutor() - app.add_routes([web.post("/", partial(handle, executor=executor))]) + app.add_routes([web.post("/", partial(handle, executor=executor()))]) return app diff --git a/contrib/python/black/blackd/middlewares.py b/contrib/python/black/blackd/middlewares.py index 75ec9267bd0..43448c2514a 100644 --- a/contrib/python/black/blackd/middlewares.py +++ b/contrib/python/black/blackd/middlewares.py @@ -1,4 +1,4 @@ -from typing import Awaitable, Callable, Iterable +from collections.abc import Awaitable, Callable, Iterable from aiohttp.typedefs import Middleware from aiohttp.web_middlewares import middleware diff --git a/contrib/python/black/blib2to3/pgen2/driver.py b/contrib/python/black/blib2to3/pgen2/driver.py index efa5a907dc8..486ac6150e6 100644 --- a/contrib/python/black/blib2to3/pgen2/driver.py +++ b/contrib/python/black/blib2to3/pgen2/driver.py @@ -21,10 +21,11 @@ import os import pkgutil import sys +from collections.abc import Iterable, Iterator from contextlib import contextmanager from dataclasses import dataclass, field from logging import Logger -from typing import IO, Any, Iterable, Iterator, Optional, Union, cast +from typing import IO, Any, Optional, Union, cast from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.tokenize import GoodTokenInfo diff --git a/contrib/python/black/blib2to3/pgen2/literals.py b/contrib/python/black/blib2to3/pgen2/literals.py index 3b219c42f93..a738c10f460 100644 --- a/contrib/python/black/blib2to3/pgen2/literals.py +++ b/contrib/python/black/blib2to3/pgen2/literals.py @@ -4,7 +4,6 @@ """Safely evaluate Python string literals without using eval().""" import re -from typing import Match simple_escapes: dict[str, str] = { "a": "\a", @@ -20,7 +19,7 @@ } -def escape(m: Match[str]) -> str: +def escape(m: re.Match[str]) -> str: all, tail = m.group(0, 1) assert all.startswith("\\") esc = simple_escapes.get(tail) diff --git a/contrib/python/black/blib2to3/pgen2/parse.py b/contrib/python/black/blib2to3/pgen2/parse.py index 2ac89c97094..10202ab6002 100644 --- a/contrib/python/black/blib2to3/pgen2/parse.py +++ b/contrib/python/black/blib2to3/pgen2/parse.py @@ -9,8 +9,9 @@ how this parsing engine works. """ +from collections.abc import Callable, Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Optional, Union, cast from blib2to3.pgen2.grammar import Grammar from blib2to3.pytree import NL, Context, Leaf, Node, RawNode, convert diff --git a/contrib/python/black/blib2to3/pgen2/pgen.py b/contrib/python/black/blib2to3/pgen2/pgen.py index 922f1069601..692e9db28f9 100644 --- a/contrib/python/black/blib2to3/pgen2/pgen.py +++ b/contrib/python/black/blib2to3/pgen2/pgen.py @@ -2,7 +2,8 @@ # Licensed to PSF under a Contributor Agreement. import os -from typing import IO, Any, Iterator, NoReturn, Optional, Sequence, Union +from collections.abc import Iterator, Sequence +from typing import IO, Any, NoReturn, Optional, Union from blib2to3.pgen2 import grammar, token, tokenize from blib2to3.pgen2.tokenize import GoodTokenInfo @@ -363,7 +364,9 @@ def raise_error(self, msg: str, *args: Any) -> NoReturn: msg = msg % args except Exception: msg = " ".join([msg] + list(map(str, args))) - raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line)) + raise SyntaxError( + msg, (str(self.filename), self.end[0], self.end[1], self.line) + ) class NFAState: diff --git a/contrib/python/black/blib2to3/pgen2/tokenize.py b/contrib/python/black/blib2to3/pgen2/tokenize.py index f7d0215c4b5..407c184dd74 100644 --- a/contrib/python/black/blib2to3/pgen2/tokenize.py +++ b/contrib/python/black/blib2to3/pgen2/tokenize.py @@ -29,7 +29,9 @@ import builtins import sys -from typing import Callable, Final, Iterable, Iterator, Optional, Pattern, Union +from collections.abc import Callable, Iterable, Iterator +from re import Pattern +from typing import Final, Optional, Union from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.token import ( @@ -219,7 +221,7 @@ def _combinations(*l: str) -> set[str]: | {f"{prefix}'" for prefix in _strprefixes | _fstring_prefixes} | {f'{prefix}"' for prefix in _strprefixes | _fstring_prefixes} ) -fstring_prefix: Final = ( +fstring_prefix: Final = tuple( {f"{prefix}'" for prefix in _fstring_prefixes} | {f'{prefix}"' for prefix in _fstring_prefixes} | {f"{prefix}'''" for prefix in _fstring_prefixes} @@ -457,7 +459,7 @@ def untokenize(iterable: Iterable[TokenInfo]) -> str: def is_fstring_start(token: str) -> bool: - return builtins.any(token.startswith(prefix) for prefix in fstring_prefix) + return token.startswith(fstring_prefix) def _split_fstring_start_and_middle(token: str) -> tuple[str, str]: diff --git a/contrib/python/black/blib2to3/pytree.py b/contrib/python/black/blib2to3/pytree.py index d2d135e7d1d..d57584685a2 100644 --- a/contrib/python/black/blib2to3/pytree.py +++ b/contrib/python/black/blib2to3/pytree.py @@ -12,7 +12,8 @@ # mypy: allow-untyped-defs, allow-incomplete-defs -from typing import Any, Iterable, Iterator, Optional, TypeVar, Union +from collections.abc import Iterable, Iterator +from typing import Any, Optional, TypeVar, Union from blib2to3.pgen2.grammar import Grammar