diff --git a/changelog/14523.improvement.rst b/changelog/14523.improvement.rst new file mode 100644 index 00000000000..f809c6803a6 --- /dev/null +++ b/changelog/14523.improvement.rst @@ -0,0 +1,4 @@ +Large assertion comparison diffs are now built lazily and capped to the +truncation budget, so a huge diff is no longer formatted in full just to +be truncated. As a result the truncation footer no longer +reports the exact number of hidden lines. diff --git a/doc/en/example/reportingdemo.rst b/doc/en/example/reportingdemo.rst index 1eef1322927..65e99a14a1f 100644 --- a/doc/en/example/reportingdemo.rst +++ b/doc/en/example/reportingdemo.rst @@ -148,7 +148,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E 1 E 1... E - E ...Full output truncated (7 lines hidden), use '-vv' to show + E ...Full output truncated, use '-vv' to show failure_demo.py:62: AssertionError _________________ TestSpecialisedExplanations.test_eq_list _________________ diff --git a/doc/en/how-to/output.rst b/doc/en/how-to/output.rst index db36a5a7206..752d0206526 100644 --- a/doc/en/how-to/output.rst +++ b/doc/en/how-to/output.rst @@ -172,7 +172,7 @@ Now we can increase pytest's verbosity: E 'banana', E 'apple',... E - E ...Full output truncated (7 lines hidden), use '-vv' to show + E ...Full output truncated, use '-vv' to show test_verbosity_example.py:8: AssertionError ____________________________ test_numbers_fail _____________________________ @@ -190,7 +190,7 @@ Now we can increase pytest's verbosity: E {'10': 10, '20': 20, '30': 30, '40': 40} E ... E - E ...Full output truncated (16 lines hidden), use '-vv' to show + E ...Full output truncated, use '-vv' to show test_verbosity_example.py:14: AssertionError ___________________________ test_long_text_fail ____________________________ diff --git a/src/_pytest/assertion/__init__.py b/src/_pytest/assertion/__init__.py index e33f8b29609..ad2454ab820 100644 --- a/src/_pytest/assertion/__init__.py +++ b/src/_pytest/assertion/__init__.py @@ -12,6 +12,8 @@ from _pytest.assertion import rewrite from _pytest.assertion import truncate from _pytest.assertion import util +from _pytest.assertion._typing import NO_TRUNCATION_BUDGET +from _pytest.assertion._typing import TruncationBudget from _pytest.assertion.rewrite import assertstate_key from _pytest.config import Config from _pytest.config import hookimpl @@ -182,12 +184,14 @@ def callbinrepr(op, left: object, right: object) -> str | None: ) for new_expl in hook_result: if new_expl: - new_expl = truncate.truncate_if_required(new_expl, item) - new_expl = [line.replace("\n", "\\n") for line in new_expl] - res = "\n~".join(new_expl) - if item.config.getvalue("assertmode") == "rewrite": - res = res.replace("%", "%%") - return res + new_expl = truncate.materialize_with_truncation(new_expl, item.config) + # A truthy-but-empty iterable materialises to [], so re-check. + if new_expl: + new_expl = [line.replace("\n", "\\n") for line in new_expl] + res = "\n~".join(new_expl) + if item.config.getvalue("assertmode") == "rewrite": + res = res.replace("%", "%%") + return res return None saved_assert_hooks = util._reprcompare, util._assertion_pass @@ -223,14 +227,29 @@ def pytest_assertrepr_compare( else: # Keep it plaintext when not using terminalrepoterer (#14377). highlighter = util.dummy_highlighter - explanation = list( - util.assertrepr_compare( - op=op, - left=left, - right=right, - verbose=config.get_verbosity(Config.VERBOSITY_ASSERTIONS), - highlighter=highlighter, - assertion_text_diff_style=util.get_assertion_text_diff_style(config), + # When truncation is going to clip the explanation downstream, cap the + # comparison helpers' formatting at what the truncator will actually pull + # (the raw limits plus the footer slack) so no effort is spent formatting + # lines/chars that would be dropped anyway. + should_truncate, base_budget = truncate._get_truncation_parameters(config) + if should_truncate: + truncation_budget = TruncationBudget( + max_lines=base_budget.max_lines + truncate.TRUNCATION_FOOTER_LINES + 1 + if base_budget.max_lines > 0 + else 0, + max_chars=base_budget.max_chars + truncate.TRUNCATION_FOOTER_CHARS + if base_budget.max_chars > 0 + else 0, ) + else: + truncation_budget = NO_TRUNCATION_BUDGET + lines = util.assertrepr_compare( + op=op, + left=left, + right=right, + verbose=config.get_verbosity(Config.VERBOSITY_ASSERTIONS), + highlighter=highlighter, + assertion_text_diff_style=util.get_assertion_text_diff_style(config), + truncation_budget=truncation_budget, ) - return explanation or None + return truncate.materialize_with_truncation(lines, config) or None diff --git a/src/_pytest/assertion/_compare_any.py b/src/_pytest/assertion/_compare_any.py index a1e729187a7..9a1e124bc5b 100644 --- a/src/_pytest/assertion/_compare_any.py +++ b/src/_pytest/assertion/_compare_any.py @@ -19,6 +19,8 @@ from _pytest.assertion._guards import istext from _pytest.assertion._typing import _AssertionTextDiffStyle from _pytest.assertion._typing import _HighlightFunc +from _pytest.assertion._typing import NO_TRUNCATION_BUDGET +from _pytest.assertion._typing import TruncationBudget from _pytest.assertion.compare_text import _compare_eq_text @@ -28,6 +30,7 @@ def _compare_eq_any( highlighter: _HighlightFunc, verbose: int, assertion_text_diff_style: _AssertionTextDiffStyle, + truncation_budget: TruncationBudget = NO_TRUNCATION_BUDGET, ) -> Iterator[str]: """Yield the per-line explanation for ``left == right`` (without summary). @@ -42,6 +45,7 @@ def _compare_eq_any( highlighter, verbose, assertion_text_diff_style, + truncation_budget, ) else: from _pytest.approx import Approx @@ -70,10 +74,14 @@ def _compare_eq_any( elif isset(left) and isset(right): yield from _compare_eq_set(left, right, highlighter, verbose) elif ismapping(left) and ismapping(right): - yield from _compare_eq_mapping(left, right, highlighter, verbose) + yield from _compare_eq_mapping( + left, right, highlighter, verbose, truncation_budget + ) if isiterable(left) and isiterable(right): - yield from _compare_eq_iterable(left, right, highlighter, verbose) + yield from _compare_eq_iterable( + left, right, highlighter, verbose, truncation_budget + ) def _compare_eq_cls( diff --git a/src/_pytest/assertion/_compare_mapping.py b/src/_pytest/assertion/_compare_mapping.py index 4edb47026c6..6380c3cacd7 100644 --- a/src/_pytest/assertion/_compare_mapping.py +++ b/src/_pytest/assertion/_compare_mapping.py @@ -1,11 +1,16 @@ from __future__ import annotations +from collections.abc import Collection from collections.abc import Iterator from collections.abc import Mapping +import heapq import pprint +from _pytest._io.pprint import _safe_key from _pytest._io.saferepr import saferepr from _pytest.assertion._typing import _HighlightFunc +from _pytest.assertion._typing import NO_TRUNCATION_BUDGET +from _pytest.assertion._typing import TruncationBudget def _compare_eq_mapping( @@ -13,6 +18,7 @@ def _compare_eq_mapping( right: Mapping[object, object], highlighter: _HighlightFunc, verbose: int = 0, + truncation_budget: TruncationBudget = NO_TRUNCATION_BUDGET, ) -> Iterator[str]: set_left = set(left) set_right = set(right) @@ -36,13 +42,31 @@ def _compare_eq_mapping( len_extra_left = len(extra_left) if len_extra_left: yield f"Left contains {len_extra_left} more item{'' if len_extra_left == 1 else 's'}:" - yield from highlighter( - pprint.pformat({k: left[k] for k in extra_left}) - ).splitlines() + yield from _format_extra_items(left, extra_left, highlighter, truncation_budget) extra_right = set_right - set_left len_extra_right = len(extra_right) if len_extra_right: yield f"Right contains {len_extra_right} more item{'' if len_extra_right == 1 else 's'}:" + yield from _format_extra_items( + right, extra_right, highlighter, truncation_budget + ) + + +def _format_extra_items( + mapping: Mapping[object, object], + keys: Collection[object], + highlighter: _HighlightFunc, + truncation_budget: TruncationBudget, +) -> Iterator[str]: + """Render the "X contains N more items" subdict.""" + max_lines = truncation_budget.max_lines + if max_lines == 0 or len(keys) <= max_lines: + # If no need to truncate, let pprint handle it. yield from highlighter( - pprint.pformat({k: right[k] for k in extra_right}) + pprint.pformat({k: mapping[k] for k in keys}) ).splitlines() + else: + # To avoid spending effort on formatting entries that would be truncated, + # only format the needed entries, keeping the sorting that pprint would use. + for k in heapq.nsmallest(max_lines, keys, key=_safe_key): + yield highlighter(saferepr({k: mapping[k]})) diff --git a/src/_pytest/assertion/_compare_sequence.py b/src/_pytest/assertion/_compare_sequence.py index 5705c45884d..cada856cc85 100644 --- a/src/_pytest/assertion/_compare_sequence.py +++ b/src/_pytest/assertion/_compare_sequence.py @@ -7,6 +7,8 @@ from _pytest._io.pprint import PrettyPrinter from _pytest._io.saferepr import saferepr from _pytest.assertion._typing import _HighlightFunc +from _pytest.assertion._typing import NO_TRUNCATION_BUDGET +from _pytest.assertion._typing import TruncationBudget from _pytest.compat import running_on_ci @@ -15,6 +17,7 @@ def _compare_eq_iterable( right: Iterable[object], highlighter: _HighlightFunc, verbose: int = 0, + truncation_budget: TruncationBudget = NO_TRUNCATION_BUDGET, ) -> Iterator[str]: if verbose <= 0 and not running_on_ci(): yield "Use -v to get more diff" @@ -22,19 +25,22 @@ def _compare_eq_iterable( # dynamic import to speedup pytest import difflib - left_formatting = PrettyPrinter().pformat(left).splitlines() - right_formatting = PrettyPrinter().pformat(right).splitlines() + pp = PrettyPrinter() + # ``pformat_lines`` spells an unbounded dimension ``None``; the budget spells it ``0``. + max_lines = truncation_budget.max_lines or None + max_chars = truncation_budget.max_chars or None + left_formatting = pp.pformat_lines(left, max_lines=max_lines, max_chars=max_chars) + right_formatting = pp.pformat_lines(right, max_lines=max_lines, max_chars=max_chars) yield "" yield "Full diff: (-: missing in left side, +: extra in left side)" # "right" is the expected base against which we compare "left", # see https://github.com/pytest-dev/pytest/issues/3333 - yield from highlighter( - "\n".join( - line.rstrip() for line in difflib.ndiff(right_formatting, left_formatting) - ), - lexer="diff", - ).splitlines() + # Highlight each ndiff line individually so the streaming truncator can + # stop pulling from ``difflib.ndiff`` once its budget is full; the diff + # lexer is line-oriented, so per-line highlighting is equivalent. + for line in difflib.ndiff(right_formatting, left_formatting): + yield highlighter(line.rstrip(), lexer="diff") def _compare_eq_sequence( diff --git a/src/_pytest/assertion/_typing.py b/src/_pytest/assertion/_typing.py index d419e09aea6..d9f0ad8e78c 100644 --- a/src/_pytest/assertion/_typing.py +++ b/src/_pytest/assertion/_typing.py @@ -21,6 +21,10 @@ class TruncationBudget: max_chars: int +# Reusable "no cap" budget, used as a default argument (B008). +NO_TRUNCATION_BUDGET = TruncationBudget(max_lines=0, max_chars=0) + + class _HighlightFunc(Protocol): # noqa: PYI046 def __call__(self, source: str, lexer: Literal["diff", "python"] = "python") -> str: """Apply highlighting to the given source.""" diff --git a/src/_pytest/assertion/compare_text.py b/src/_pytest/assertion/compare_text.py index 08e85fc5811..9bee86a0b93 100644 --- a/src/_pytest/assertion/compare_text.py +++ b/src/_pytest/assertion/compare_text.py @@ -5,6 +5,8 @@ from _pytest._io.saferepr import saferepr from _pytest.assertion._typing import _AssertionTextDiffStyle from _pytest.assertion._typing import _HighlightFunc +from _pytest.assertion._typing import NO_TRUNCATION_BUDGET +from _pytest.assertion._typing import TruncationBudget from _pytest.assertion.highlight import dummy_highlighter from _pytest.compat import assert_never @@ -15,12 +17,13 @@ def _compare_eq_text( highlighter: _HighlightFunc, verbose: int, assertion_text_diff_style: _AssertionTextDiffStyle, + truncation_budget: TruncationBudget = NO_TRUNCATION_BUDGET, ) -> Iterator[str]: match assertion_text_diff_style: case "block": yield from _diff_text_block(left, right) case "ndiff": - yield from _diff_text(left, right, highlighter, verbose) + yield from _diff_text(left, right, highlighter, verbose, truncation_budget) case unreachable: assert_never(unreachable) @@ -39,12 +42,20 @@ def _format_text_block_lines(text: str) -> Iterator[str]: def _diff_text( - left: str, right: str, highlighter: _HighlightFunc, verbose: int = 0 + left: str, + right: str, + highlighter: _HighlightFunc, + verbose: int = 0, + truncation_budget: TruncationBudget = NO_TRUNCATION_BUDGET, ) -> Iterator[str]: """Yield the explanation for the diff between text. Unless --verbose is used this will skip leading and trailing characters which are identical to keep the diff minimal. + + When a truncation budget is set, the inputs to ``ndiff`` are capped + first, so the truncated head may differ from the head of the + unbounded diff. """ from difflib import ndiff @@ -75,23 +86,42 @@ def _diff_text( left = repr(str(left)) right = repr(str(right)) yield "Strings contain only whitespace, escaping them using repr()" + left_lines = _cap_ndiff_input(left, keepends, truncation_budget) + right_lines = _cap_ndiff_input(right, keepends, truncation_budget) # "right" is the expected base against which we compare "left", # see https://github.com/pytest-dev/pytest/issues/3333 yield from highlighter( - "\n".join( - line.strip("\n") - for line in ndiff(right.splitlines(keepends), left.splitlines(keepends)) - ), + "\n".join(line.strip("\n") for line in ndiff(right_lines, left_lines)), lexer="diff", ).splitlines() -def _notin_text(term: str, text: str, verbose: int = 0) -> Iterator[str]: +def _cap_ndiff_input(text: str, keepends: bool, budget: TruncationBudget) -> list[str]: + """Cap an ``ndiff`` input to the truncation budget, as split lines. + + A char slice first (bounds a few huge lines, whose intraline diff is + O(len^2)), then a line slice (bounds many lines). A ``0`` limit leaves + that dimension unbounded. + """ + if budget.max_chars > 0: + text = text[: budget.max_chars] + lines = text.splitlines(keepends) + if budget.max_lines > 0: + lines = lines[: budget.max_lines] + return lines + + +def _notin_text( + term: str, + text: str, + verbose: int = 0, + truncation_budget: TruncationBudget = NO_TRUNCATION_BUDGET, +) -> Iterator[str]: index = text.find(term) head = text[:index] tail = text[index + len(term) :] correct_text = head + tail - diff = _diff_text(text, correct_text, dummy_highlighter, verbose) + diff = _diff_text(text, correct_text, dummy_highlighter, verbose, truncation_budget) yield f"{saferepr(term, maxsize=42)} is contained here:" for line in diff: if line.startswith("Skipping"): diff --git a/src/_pytest/assertion/truncate.py b/src/_pytest/assertion/truncate.py index 30cff154770..4627b7c6fe1 100644 --- a/src/_pytest/assertion/truncate.py +++ b/src/_pytest/assertion/truncate.py @@ -6,39 +6,73 @@ from __future__ import annotations +from collections.abc import Iterable + from _pytest.assertion._typing import TruncationBudget from _pytest.compat import running_on_ci from _pytest.config import Config -from _pytest.nodes import Item DEFAULT_MAX_LINES = 8 DEFAULT_MAX_CHARS = DEFAULT_MAX_LINES * 80 USAGE_MSG = "use '-vv' to show" +TRUNCATION_MSG = f"...Full output truncated, {USAGE_MSG}" + +# Truncating appends a footer: ``...`` on the last kept line, a blank line, +# then ``TRUNCATION_MSG``. A body may exceed the raw budget by the footer's +# cost before truncating, so a body that nearly fits is not cut just to +# make room for the footer. +TRUNCATION_FOOTER_LINES = 2 # blank separator + message line +TRUNCATION_FOOTER_CHARS = len("...") + len(TRUNCATION_MSG) + +def materialize_with_truncation(lines: Iterable[str], config: Config) -> list[str]: + """Materialise a streaming explanation, applying truncation lazily. + + Pulls from ``lines`` only until the truncation threshold is reached; + the rest of the iterator is dropped without being consumed. + """ + should_truncate, budget = _get_truncation_parameters(config) + if not should_truncate: + return list(lines) + + tolerable_max_chars = budget.max_chars + TRUNCATION_FOOTER_CHARS + # Pull one line past the point where ``_truncate_explanation`` keeps the + # body whole (``max_lines + TRUNCATION_FOOTER_LINES``) so it can detect the + # overflow, without us materialising more than we need. + line_cap = ( + budget.max_lines + TRUNCATION_FOOTER_LINES + 1 if budget.max_lines > 0 else None + ) + buffered: list[str] = [] + char_count = 0 + for line in lines: + buffered.append(line) + char_count += len(line) + if line_cap is not None and len(buffered) >= line_cap: + break + if budget.max_chars > 0 and char_count > tolerable_max_chars: + break + else: + # Iterator exhausted within limits — nothing to truncate. + return buffered -def truncate_if_required(explanation: list[str], item: Item) -> list[str]: - """Truncate this assertion explanation if the given test item is eligible.""" - should_truncate, budget = _get_truncation_parameters(item) - if should_truncate: - return _truncate_explanation(explanation, budget) - return explanation + return _truncate_explanation(buffered, budget) -def _get_truncation_parameters(item: Item) -> tuple[bool, TruncationBudget]: - """Return the truncation parameters related to the given item, as (should truncate, budget).""" +def _get_truncation_parameters(config: Config) -> tuple[bool, TruncationBudget]: + """Return the truncation parameters from the given config, as (should truncate, budget).""" # We do not need to truncate if one of conditions is met: # 1. Verbosity level is 2 or more; # 2. Test is being run in CI environment; # 3. Both truncation_limit_lines and truncation_limit_chars # .ini parameters are set to 0 explicitly. - max_lines = item.config.getini("truncation_limit_lines") + max_lines = config.getini("truncation_limit_lines") max_lines = int(max_lines if max_lines is not None else DEFAULT_MAX_LINES) - max_chars = item.config.getini("truncation_limit_chars") + max_chars = config.getini("truncation_limit_chars") max_chars = int(max_chars if max_chars is not None else DEFAULT_MAX_CHARS) - verbose = item.config.get_verbosity(Config.VERBOSITY_ASSERTIONS) + verbose = config.get_verbosity(Config.VERBOSITY_ASSERTIONS) should_truncate = verbose < 2 and not running_on_ci() should_truncate = should_truncate and (max_lines > 0 or max_chars > 0) @@ -62,22 +96,13 @@ def _truncate_explanation( When this function is launched we know max_lines > 0 or max_chars > 0 because _get_truncation_parameters was called first. """ - # The length of the truncation explanation depends on the number of lines - # removed but is at least 68 characters: - # The real value is - # 64 (for the base message: - # '...\n...Full output truncated (1 line hidden), use '-vv' to show")' - # ) - # + 1 (for plural) - # + int(math.log10(len(input_lines) - max_lines)) (number of hidden line, at least 1) - # + 3 for the '...' added to the truncated line - # But if there's more than 100 lines it's very likely that we're going to - # truncate, so we don't need the exact value using log10. - tolerable_max_chars = ( - budget.max_chars + 70 # 64 + 1 (for plural) + 2 (for '99') + 3 for '...' - ) - # The truncation explanation add two lines to the output - if budget.max_lines == 0 or len(input_lines) <= budget.max_lines + 2: + # ``max_chars`` bounds the body only; the footer slack is added on top + # (see ``TRUNCATION_FOOTER_CHARS``). + tolerable_max_chars = budget.max_chars + TRUNCATION_FOOTER_CHARS + if ( + budget.max_lines == 0 + or len(input_lines) <= budget.max_lines + TRUNCATION_FOOTER_LINES + ): if ( budget.max_chars == 0 or sum(len(s) for s in input_lines) <= tolerable_max_chars @@ -88,24 +113,19 @@ def _truncate_explanation( # Truncate first to max_lines, and then truncate to max_chars if necessary truncated_explanation = input_lines[: budget.max_lines] # We reevaluate the need to truncate chars following removal of some lines - need_to_truncate_char = ( + if ( budget.max_chars > 0 and sum(len(e) for e in truncated_explanation) > tolerable_max_chars - ) - if need_to_truncate_char: + ): truncated_explanation = _truncate_by_char_count( truncated_explanation, budget.max_chars ) # Something was truncated, adding '...' at the end to show that truncated_explanation[-1] += "..." - truncated_line_count = ( - len(input_lines) - len(truncated_explanation) + int(need_to_truncate_char) - ) return [ *truncated_explanation, "", - f"...Full output truncated ({truncated_line_count} line" - f"{'' if truncated_line_count == 1 else 's'} hidden), {USAGE_MSG}", + TRUNCATION_MSG, ] diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 5e5ef543c13..3898a6f79a8 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -19,6 +19,8 @@ from _pytest.assertion._guards import istext from _pytest.assertion._typing import _AssertionTextDiffStyle from _pytest.assertion._typing import _HighlightFunc +from _pytest.assertion._typing import NO_TRUNCATION_BUDGET +from _pytest.assertion._typing import TruncationBudget from _pytest.assertion.compare_text import _notin_text from _pytest.assertion.highlight import dummy_highlighter as dummy_highlighter from _pytest.config import Config @@ -140,6 +142,7 @@ def assertrepr_compare( verbose: int, highlighter: _HighlightFunc, assertion_text_diff_style: _AssertionTextDiffStyle, + truncation_budget: TruncationBudget = NO_TRUNCATION_BUDGET, ) -> Iterator[str]: """Yield specialised explanations for some operators/operands. @@ -183,9 +186,10 @@ def assertrepr_compare( highlighter, verbose, assertion_text_diff_style, + truncation_budget, ) elif op == "not in" and istext(left) and istext(right): - source = _notin_text(left, right, verbose) + source = _notin_text(left, right, verbose, truncation_budget) elif op in {"!=", ">=", "<=", ">", "<"} and isset(left) and isset(right): source = SET_COMPARISON_FUNCTIONS[op](left, right, highlighter, verbose) else: diff --git a/testing/python/approx.py b/testing/python/approx.py index fd3533592f5..64e3f39c5f1 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -314,7 +314,7 @@ def test_error_messages_with_different_verbosity(self, assert_approx_raises_rege rf"^ \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}e-{SOME_INT}$", rf"^ \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}e-{SOME_INT}\.\.\.$", "^ $", - rf"^ ...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show$", + r"^ ...Full output truncated, use '-vv' to show$", ], verbosity_level=0, ) diff --git a/testing/test_assertion.py b/testing/test_assertion.py index 2da4fb94847..609c4b1a62f 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -17,8 +17,11 @@ from _pytest.assertion import truncate from _pytest.assertion import util from _pytest.assertion._compare_any import _compare_eq_cls +from _pytest.assertion._compare_mapping import _compare_eq_mapping +from _pytest.assertion._typing import NO_TRUNCATION_BUDGET from _pytest.assertion._typing import TruncationBudget from _pytest.assertion.compare_text import _compare_eq_text +from _pytest.assertion.compare_text import _notin_text from _pytest.config import Config as _Config from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester @@ -29,6 +32,9 @@ def mock_config( verbose: int = 0, assertion_override: int | None = None, assertion_text_diff_style: str = util.ASSERTION_TEXT_DIFF_STYLE_NDIFF, + truncation_limit_lines: str = "0", + truncation_limit_chars: str = "0", + has_terminalreporter: bool = True, ): class TerminalWriter: def _highlight(self, source, lexer="python"): @@ -36,12 +42,18 @@ def _highlight(self, source, lexer="python"): class PluginManager: def has_plugin(self, name: str) -> bool: - return True + return has_terminalreporter class Config: pluginmanager = PluginManager() def get_terminal_writer(self): + # When the terminalreporter plugin is absent the dispatcher must + # fall back to the plaintext highlighter without reaching for a + # terminal writer (#14377); make that misuse fail loudly. + assert has_terminalreporter, ( + "get_terminal_writer() must not be called without terminalreporter" + ) return TerminalWriter() def get_verbosity(self, verbosity_type: str | None = None) -> int: @@ -57,6 +69,13 @@ def get_verbosity(self, verbosity_type: str | None = None) -> int: def getini(self, name: str) -> str: if name == util.ASSERTION_TEXT_DIFF_STYLE_INI: return assertion_text_diff_style + # Truncation defaults to disabled (``"0"``) so ``callop``-style + # tests can compare against the full explanation; the dispatcher + # tests pass explicit limits to exercise the budget wiring. + if name == "truncation_limit_lines": + return truncation_limit_lines + if name == "truncation_limit_chars": + return truncation_limit_chars raise KeyError(f"Not mocked out: {name}") return Config() @@ -492,6 +511,62 @@ def test_text_diff_ndiff_style(self) -> None: "+ spam", ] + def test_text_diff_budget_caps_ndiff_input(self) -> None: + """A truncation budget caps the inputs to ndiff, so the result is + bounded instead of growing with the input.""" + left = "\n".join(f"left {i}" for i in range(1000)) + right = "\n".join(f"right {i}" for i in range(1000)) + ndiff_style = util.ASSERTION_TEXT_DIFF_STYLE_NDIFF + capped = list( + _compare_eq_text( + left, + right, + util.dummy_highlighter, + 1, + ndiff_style, + TruncationBudget(max_lines=11, max_chars=710), + ) + ) + full = list( + _compare_eq_text( + left, + right, + util.dummy_highlighter, + 1, + ndiff_style, + NO_TRUNCATION_BUDGET, + ) + ) + assert len(capped) < 80 + assert len(full) > 1500 + # a few huge lines: the char budget bounds each emitted line. + capped_chars = list( + _compare_eq_text( + "x" * 100_000, + "y" * 100_000, + util.dummy_highlighter, + 1, + ndiff_style, + TruncationBudget(max_lines=11, max_chars=710), + ) + ) + assert all(len(line) < 1000 for line in capped_chars) + + def test_notin_text_budget_caps_ndiff_input(self) -> None: + """``not in`` runs the same ndiff machinery as ``==`` and is capped + the same way.""" + # The needle sits in the middle so an uncapped diff has to walk + # past ~N identical chars to reach it. + needle = "NEEDLE" + text = "a" * 100_000 + needle + "a" * 100_000 + capped = list( + _notin_text(needle, text, 1, TruncationBudget(max_lines=11, max_chars=710)) + ) + full = list(_notin_text(needle, text, 1, NO_TRUNCATION_BUDGET)) + assert len(capped) < 80 + assert all(len(line) < 1000 for line in capped) + assert sum(len(line) for line in full) > 100_000 + def test_text_skipping(self) -> None: lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs") assert lines is not None @@ -913,6 +988,35 @@ def test_dict_different_items(self) -> None: " }", ] + def test_dict_extra_items_bounded_under_budget(self) -> None: + """With more extra keys than the budget, only the smallest + ``max_lines`` keys are emitted, one per line.""" + out = list( + _compare_eq_mapping( + {i: i for i in range(1000)}, + {}, + util.dummy_highlighter, + 0, + TruncationBudget(max_lines=5, max_chars=350), + ) + ) + assert out[0] == "Left contains 1000 more items:" + body = out[1:] + assert body == [f"{{{i}: {i}}}" for i in range(5)] # smallest 5, sorted + + def test_dict_extra_items_small_keeps_pformat_block(self) -> None: + """Under the budget, the compact pprint block is unchanged.""" + out = list( + _compare_eq_mapping( + {"b": 2, "a": 1}, + {}, + util.dummy_highlighter, + 0, + TruncationBudget(max_lines=5, max_chars=350), + ) + ) + assert out == ["Left contains 2 more items:", "{'a': 1, 'b': 2}"] + def test_mapping_different_items(self) -> None: class SimpleMapping(Mapping[str, int]): def __init__(self, values: dict[str, int]) -> None: @@ -1165,7 +1269,7 @@ def test_recursive_dataclasses(self, pytester: Pytester) -> None: "E Drill down into differing attribute g:", "E g: S(a=10, b='ten') != S(a=20, b='xxx')...", "E ", - "E ...Full output truncated (51 lines hidden), use '-vv' to show", + "E ...Full output truncated, use '-vv' to show", ], consecutive=True, ) @@ -1544,7 +1648,6 @@ def test_truncates_at_8_lines_when_given_list_of_empty_strings(self) -> None: assert result != expl assert len(result) == 8 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert "42 lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1557,7 +1660,6 @@ def test_truncates_at_8_lines_when_first_8_lines_are_LT_max_chars(self) -> None: assert result != expl assert len(result) == 8 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert f"{total_lines - 8} lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1580,7 +1682,7 @@ def test_truncates_full_line_because_of_max_chars(self) -> None: "a" * 10, "...", "", - "...Full output truncated (1 line hidden), use '-vv' to show", + "...Full output truncated, use '-vv' to show", ] def test_truncates_edgecase_when_truncation_message_makes_the_result_longer_for_chars( @@ -1611,7 +1713,6 @@ def test_truncates_at_8_lines_when_first_8_lines_are_EQ_max_chars(self) -> None: assert result != expl assert len(result) == 16 - 8 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert "8 lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1623,7 +1724,6 @@ def test_truncates_at_4_lines_when_first_4_lines_are_GT_max_chars(self) -> None: assert result != expl assert len(result) == 4 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert "7 lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1635,7 +1735,6 @@ def test_truncates_at_1_line_when_first_line_is_GT_max_chars(self) -> None: assert result != expl assert len(result) == 1 + self.LINES_IN_TRUNCATION_MSG assert "Full output truncated" in result[-1] - assert "1000 lines hidden" in result[-1] last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] assert last_line_before_trunc_msg.endswith("...") @@ -1643,7 +1742,6 @@ def test_full_output_truncated(self, monkeypatch, pytester: Pytester) -> None: """Test against full runpytest() output.""" line_count = 7 line_len = 100 - expected_truncated_lines = 2 pytester.makepyfile( rf""" def test_many_lines(): @@ -1662,7 +1760,7 @@ def test_many_lines(): [ "*+ 1*", "*+ 3*", - f"*truncated ({expected_truncated_lines} lines hidden)*use*-vv*", + "*Full output truncated*use*-vv*", ] ) @@ -1676,7 +1774,7 @@ def test_many_lines(): [ "*+ 1*", "*+ 3*", - f"*truncated ({expected_truncated_lines} lines hidden)*use*-vv*", + "*Full output truncated*use*-vv*", ] ) @@ -1691,7 +1789,7 @@ def test_many_lines(): (4, None, 0), (0, None, 0), (None, 8, 6), - (None, 9, 0), + (None, 33, 0), (None, 0, 0), (0, 0, 0), (0, 1000, 0), @@ -1718,7 +1816,7 @@ def test(): # This test produces 6 lines of diff output or 79 characters # So the effect should be when threshold is < 4 lines (considering 2 additional lines for explanation) - # Or < 9 characters (considering 70 additional characters for explanation) + # Or < 33 characters (considering the ~46-char footer slack, see truncate.TRUNCATION_FOOTER_CHARS) monkeypatch.delenv("CI", raising=False) @@ -1732,9 +1830,7 @@ def test(): result = pytester.runpytest() if expected_lines_hidden != 0: - result.stdout.fnmatch_lines( - [f"*truncated ({expected_lines_hidden} lines hidden)*"] - ) + result.stdout.fnmatch_lines(["*Full output truncated*"]) else: result.stdout.no_fnmatch_line("*truncated*") result.stdout.fnmatch_lines( @@ -1745,6 +1841,257 @@ def test(): ) +class TestMaterializeWithTruncation: + """Tests for ``truncate.materialize_with_truncation``.""" + + @staticmethod + def _config_with_limits(verbose: int = 0): + class C: + def getini(self, name: str) -> object: + return None # use defaults (8 lines / 640 chars) + + def get_verbosity(self, _verbosity_type: str | None = None) -> int: + return verbose + + return C() + + def test_sized_input_returns_same_shape_as_iterator_input(self) -> None: + """A list input truncates the same way as an iterator over it.""" + content = [f"line {i}" for i in range(50)] + sized = truncate.materialize_with_truncation( + content, self._config_with_limits() + ) + unsized = truncate.materialize_with_truncation( + iter(content), self._config_with_limits() + ) + assert sized[0] == unsized[0] == "line 0" + assert any("truncated" in line for line in sized) + assert any("truncated" in line for line in unsized) + + def test_truncation_disabled_returns_full_input(self) -> None: + """Verbose >= 2 disables truncation; the iterator is fully drained.""" + lines = (f"line {i}" for i in range(50)) + result = truncate.materialize_with_truncation( + lines, self._config_with_limits(verbose=2) + ) + assert result == [f"line {i}" for i in range(50)] + assert not any("truncated" in line for line in result) + + def test_idempotent_on_already_truncated_list(self) -> None: + """Re-truncating an already-truncated explanation changes nothing. + + The dispatcher re-applies truncation to the built-in impl's + already-truncated output. + """ + once = truncate.materialize_with_truncation( + (f"line {i}" for i in range(200)), self._config_with_limits() + ) + twice = truncate.materialize_with_truncation(once, self._config_with_limits()) + assert twice == once + + def test_does_not_over_consume_the_stream(self) -> None: + """Laziness guard: the input iterator is never drained past the + truncation threshold.""" + pulled = 0 + + def tripwire() -> Iterator[str]: + nonlocal pulled + for i in range(10_000): # pragma: no branch + pulled += 1 + yield f"line {i}" + raise AssertionError( # pragma: no cover + "materialize_with_truncation drained the whole stream — " + "the explanation iterator is no longer consumed lazily" + ) + + result = truncate.materialize_with_truncation( + tripwire(), self._config_with_limits() + ) + assert any("truncated" in line for line in result) + assert pulled < 20 + + def test_pull_count_is_independent_of_input_size(self) -> None: + """The number of lines pulled to truncate does not grow with the input.""" + + def count_pulls(n: int) -> int: + pulled = 0 + + def gen() -> Iterator[str]: + nonlocal pulled + for i in range(n): # pragma: no branch + pulled += 1 + yield f"line {i}" + + truncate.materialize_with_truncation(gen(), self._config_with_limits()) + return pulled + + assert count_pulls(100) == count_pulls(100_000) + + def _explain_capped(self, left: object, right: object) -> list[str]: + # Drive the comparison as the dispatcher does: budget-capped + # ``assertrepr_compare``, then materialise with truncation. + config = self._config_with_limits() + should, base = truncate._get_truncation_parameters(config) + cap = ( + TruncationBudget( + max_lines=base.max_lines + 3 if base.max_lines > 0 else 0, + max_chars=base.max_chars + 70 if base.max_chars > 0 else 0, + ) + if should + else NO_TRUNCATION_BUDGET + ) + src = util.assertrepr_compare( + op="==", + left=left, + right=right, + verbose=1, + highlighter=util.dummy_highlighter, + assertion_text_diff_style=util.ASSERTION_TEXT_DIFF_STYLE_NDIFF, + truncation_budget=cap, + ) + return truncate.materialize_with_truncation(src, config) + + @pytest.mark.parametrize("shape", ["list", "tuple", "dict", "set"]) + def test_formatting_work_is_bounded_for_a_10_line_display(self, shape: str) -> None: + """A huge comparison whose display is truncated to ~10 lines must + not format the whole input to get there. + + Element ``repr`` calls are counted as a deterministic proxy for + the formatting work (wall-clock would flake in CI). + """ + + class Tracked: + reprs = 0 + + def __init__(self, v: int) -> None: + self.v = v + + def __repr__(self) -> str: + Tracked.reprs += 1 + return f"T({self.v})" + + def __eq__(self, o: object) -> bool: + return isinstance(o, Tracked) and self.v == o.v + + def __hash__(self) -> int: + return hash(self.v) + + def make(n: int) -> tuple[object, object]: + # Two near-identical containers differing in one spot, so the + # explanation is a real (truncated) diff. + if shape in ("list", "tuple"): + seq_left = [Tracked(i) for i in range(n)] + seq_right = [Tracked(i) for i in range(n)] + seq_right[0] = Tracked(-1) + if shape == "tuple": + return tuple(seq_left), tuple(seq_right) + return seq_left, seq_right + if shape == "dict": + map_left = {i: Tracked(i) for i in range(n)} + map_right = {i: Tracked(i) for i in range(n)} + map_right[0] = Tracked(-1) + return map_left, map_right + # set + set_left = {Tracked(i) for i in range(n)} + set_right = {Tracked(i) for i in range(n)} + set_right.discard(Tracked(0)) + set_right.add(Tracked(-1)) + return set_left, set_right + + def work(n: int) -> tuple[int, int]: + left, right = make(n) + Tracked.reprs = 0 # count only the formatting, not the construction + out = self._explain_capped(left, right) + assert any("truncated" in line for line in out) + return len(out), Tracked.reprs + + lines_small, reprs_small = work(1_000) + lines_big, reprs_big = work(100_000) + assert lines_small == lines_big + assert reprs_small == reprs_big + assert reprs_big < 200 # nowhere near the 100_000-element input + + +class TestAssertReprCompareDispatcher: + """Tests for the budget wiring in ``plugin.pytest_assertrepr_compare``. + + Unlike ``callequal``/``callop``, these tests pass explicit truncation + limits, covering the branch that derives a :class:`TruncationBudget` + for the comparison helpers. + """ + + def test_both_limits_truncate_without_formatting_everything(self) -> None: + """A huge list comparison is truncated to a few lines without the + element ``repr`` work scaling with the input.""" + + class Tracked: + reprs = 0 + + def __init__(self, v: int) -> None: + self.v = v + + def __repr__(self) -> str: + Tracked.reprs += 1 + return f"T({self.v})" + + def __eq__(self, o: object) -> bool: + return isinstance(o, Tracked) and self.v == o.v + + config = mock_config( + verbose=1, # -v: emit the full diff that truncation then clips + truncation_limit_lines="4", + truncation_limit_chars="160", + ) + left = [Tracked(i) for i in range(50_000)] + right = [Tracked(i) for i in range(50_000)] + right[0] = Tracked(-1) + Tracked.reprs = 0 # count only the formatting, not the construction + result = plugin.pytest_assertrepr_compare(config, "==", left, right) + assert result is not None + assert len(result) < 20 + assert any("truncated" in line for line in result) + assert Tracked.reprs < 200 # nowhere near the 50k-element input + + def test_chars_only_budget_still_truncates(self) -> None: + """With the line limit disabled, the char cap alone still bounds + the explanation.""" + config = mock_config( + verbose=1, # -v: emit the full diff that truncation then clips + truncation_limit_lines="0", + truncation_limit_chars="160", + ) + left = list(range(1000)) + right = list(range(1, 1001)) + result = plugin.pytest_assertrepr_compare(config, "==", left, right) + assert result is not None + assert any("truncated" in line for line in result) + assert sum(len(line) for line in result) < 1000 + + def test_lines_only_budget_still_truncates(self) -> None: + """With the char limit disabled, the line cap alone still bounds + the explanation.""" + config = mock_config( + verbose=1, # -v: emit the full diff that truncation then clips + truncation_limit_lines="4", + truncation_limit_chars="0", + ) + left = list(range(1000)) + right = list(range(1, 1001)) + result = plugin.pytest_assertrepr_compare(config, "==", left, right) + assert result is not None + assert len(result) < 20 + assert any("truncated" in line for line in result) + + def test_no_terminalreporter_uses_plaintext_highlighter(self) -> None: + """Without the terminalreporter plugin the dispatcher uses the + plaintext highlighter and never touches a terminal writer (#14377).""" + config = mock_config(has_terminalreporter=False) + result = plugin.pytest_assertrepr_compare(config, "==", {1, 2}, {1, 3}) + assert result is not None + assert any("Extra items" in line for line in result) + assert not any("\x1b[" in line for line in result) + + def test_python25_compile_issue257(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -2270,6 +2617,98 @@ def raise_exit(obj): callequal(1, 1) +def test_plugin_hook_returning_none_is_skipped(pytester: Pytester) -> None: + """A ``pytest_assertrepr_compare`` impl returning ``None`` is skipped + so the next impl (or the built-in) can produce the explanation.""" + pytester.makeconftest( + """ + def pytest_assertrepr_compare(op, left, right): + # Always defer to the next plugin / the built-in. + return None + """ + ) + pytester.makepyfile( + """ + def test_diff(): + assert {1, 2} == {1, 3} + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*Extra items in the left set:*", "*Extra items in the right set:*"] + ) + + +def test_plugin_hook_returning_empty_iterator_is_skipped(pytester: Pytester) -> None: + """A plugin returning a truthy but ultimately empty iterable is + skipped after materialisation.""" + pytester.makeconftest( + """ + def pytest_assertrepr_compare(op, left, right): + return iter([]) + """ + ) + pytester.makepyfile( + """ + def test_diff(): + assert {1, 2} == {1, 3} + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*Extra items in the left set:*", "*Extra items in the right set:*"] + ) + + +def test_callbinrepr_falls_through_when_all_hooks_return_none( + pytester: Pytester, +) -> None: + """When no ``pytest_assertrepr_compare`` impl produces an explanation, + the plain assert rewrite is shown.""" + pytester.makepyfile( + """ + def test_trivial(): + assert 1 == 2 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*assert 1 == 2*"]) + result.assert_outcomes(failed=1) + + +def test_callbinrepr_plain_assert_mode(pytester: Pytester) -> None: + """In ``--assert=plain`` mode the comparison explanation is still produced.""" + pytester.makepyfile( + """ + def test_diff(): + assert {1, 2} == {1, 3} + """ + ) + result = pytester.runpytest("--assert=plain") + result.stdout.fnmatch_lines( + ["*Extra items in the left set:*", "*Extra items in the right set:*"] + ) + + +def test_exception_before_first_yield_emits_summary_and_notice(monkeypatch) -> None: + """A comparator raising before any explanation line is yielded still + produces the summary, followed by the failure notice.""" + from _pytest.assertion import _compare_any + + def raise_value_error(obj): + raise ValueError("synthetic repr failure") + + # ``istext`` is called before the first yield, so this triggers the + # failure path on the very first ``next()``. + monkeypatch.setattr(_compare_any, "istext", raise_value_error) + + expl = callequal(1, 1) + assert expl is not None + assert expl[0] == "1 == 1" + # Wording deliberately not asserted, only the underlying error's signature. + assert any("ValueError" in line or "synthetic" in line for line in expl) + + def test_assertion_location_with_coverage(pytester: Pytester) -> None: """This used to report the wrong location when run with coverage (#5754).""" p = pytester.makepyfile( @@ -2316,8 +2755,8 @@ def test(): """, [ "{bold}{red}E At index 1 diff: {reset}{number}1{hl-reset}{endline} != {reset}{number}2*", - "{bold}{red}E {light-red}- 2,{hl-reset}{endline}{reset}", - "{bold}{red}E {light-green}+ 1,{hl-reset}{endline}{reset}", + "{bold}{red}E {reset}{light-red}- 2,{hl-reset}{endline}{reset}", + "{bold}{red}E {reset}{light-green}+ 1,{hl-reset}{endline}{reset}", ], ), ( @@ -2335,8 +2774,8 @@ def test(): "{bold}{red}E Right contains 1 more item:{reset}", "{bold}{red}E {reset}{{{str}'{hl-reset}{str}number-is-0{hl-reset}{str}'{hl-reset}: {number}0*", "{bold}{red}E {reset}{light-gray} {hl-reset} {{{endline}{reset}", - "{bold}{red}E {light-gray} {hl-reset} 'number-is-1': 1,{endline}{reset}", - "{bold}{red}E {light-green}+ 'number-is-5': 5,{hl-reset}{endline}{reset}", + "{bold}{red}E {reset}{light-gray} {hl-reset} 'number-is-1': 1,{endline}{reset}", + "{bold}{red}E {reset}{light-green}+ 'number-is-5': 5,{hl-reset}{endline}{reset}", ], ), (