run_sanity tests left failing dues to AI gaslighting

This commit is contained in:
nvaldez
2026-09-23 22:56:48 +02:00
parent e750a3dca9
commit cb7c3d8cbd
5 changed files with 131 additions and 29 deletions
-5
View File
@@ -84,11 +84,6 @@ def run_pytests():
if counts["skipped"]:
detail += f", {counts['skipped']} skipped"
lines.append(f" - {file_name:<{name_width}} {detail}")
for failure in failures_by_file.get(file_name, []):
parts = failure["id"].split("::")
test_name = " : ".join(parts[1:]) if len(parts) > 1 else failure["id"]
lines.append(f" - {test_name}")
lines.append(f" - {failure.get('assert') or failure['message']}")
failed_tests = result.get("failed_tests", [])
if failed_tests:
+23 -8
View File
@@ -35,7 +35,9 @@ class TestRunner:
The counts come from the final summary line, so the last match
wins: assertion diffs printed by failing tests can contain
earlier `N passed` strings that must not be counted.
earlier `N passed` strings that must not be counted. Errored
tests (fixture or collection errors) count as failures so the
totals match the per-file listing.
"""
def last_count(pattern: str) -> int:
matches = re.findall(pattern, output)
@@ -43,7 +45,7 @@ class TestRunner:
return {
"passed": last_count(r'(\d+) passed'),
"failed": last_count(r'(\d+) failed'),
"failed": last_count(r'(\d+) failed') + last_count(r'(\d+) errors?'),
"skipped": last_count(r'(\d+) skipped'),
}
@@ -80,12 +82,14 @@ class TestRunner:
return file_coverage
def _parse_failed_tests_output(self, output: str) -> list[dict]:
"""Parse the pytest output into failed test details.
"""Parse the pytest output into failed and errored test details.
Combines the short summary (test id and exception line) with the
FAILURES traceback section (assert location and explanation) so
reports show what failed, where, and why. Failures are paired
with their traceback blocks by order.
reports show what failed, where, and why. Errored tests (fixture
or collection errors, reported as `ERROR` lines) are included
with their id and message so they do not vanish from the report.
Failures are paired with their traceback blocks by order.
Args:
output: The pytest output text.
@@ -96,21 +100,32 @@ class TestRunner:
the failing assert), and the `assert` explanation.
"""
failures: list[dict] = []
pattern = re.compile(r"^FAILED (\S+) - (.+)$", re.MULTILINE)
for match in pattern.finditer(output):
errors: list[dict] = []
failed_pattern = re.compile(r"^FAILED (\S+) - (.+)$", re.MULTILINE)
error_pattern = re.compile(r"^ERROR (\S+) - (.+)$", re.MULTILINE)
for match in failed_pattern.finditer(output):
failures.append({
"id": match.group(1),
"message": match.group(2),
"location": None,
"assert": None,
})
for match in error_pattern.finditer(output):
errors.append({
"id": match.group(1),
"message": match.group(2),
"location": None,
"assert": None,
})
# traceback details only exist for FAILURES section entries, so
# they pair with the FAILED entries, never with the errored ones
details = self._parse_failures_section(output)
if len(details) == len(failures):
for failure, detail in zip(failures, details):
failure["location"] = detail["location"]
failure["assert"] = detail["assert"]
return failures
return failures + errors
def _parse_failures_section(self, output: str) -> list[dict]:
"""Parse the FAILURES traceback section into per-test details.
+55 -15
View File
@@ -663,7 +663,31 @@ class TestVerdict:
class TestFailedTestReportIntegration:
"""End-to-end check that a real failing test reaches the report."""
"""End-to-end checks that real failures reach the report."""
@staticmethod
def _patched_runner_for(test_file):
"""Patch run_sanity.TestRunner with a real runner for a test file.
The runner runs pytest in a real subprocess (no mocks, no coverage)
against the given file so the whole chain is exercised: real pytest
output, the failure parsing, and the report formatting in
run_pytests.
"""
class TargetedRunner(TestRunner):
"""Real TestRunner redirected to the given test file."""
def __init__(self, target):
super().__init__()
self.target = target
def run_pytest(self, test_path, verbose=False, capture_output=False,
with_coverage=True, ignore=None):
return super().run_pytest(
self.target, verbose, capture_output, with_coverage=False
)
return patch("run_sanity.TestRunner", lambda: TargetedRunner(test_file))
def test_real_failure_shows_function_and_assert(self, tmp_path):
"""Test a real pytest failure nests its name and assert under the file.
@@ -680,20 +704,7 @@ class TestFailedTestReportIntegration:
encoding="utf-8",
)
class TargetedRunner(TestRunner):
"""Real TestRunner redirected to the temporary test file."""
def __init__(self, target):
super().__init__()
self.target = target
def run_pytest(self, test_path, verbose=False, capture_output=False,
with_coverage=True, ignore=None):
return super().run_pytest(
self.target, verbose, capture_output, with_coverage=False
)
with patch("run_sanity.TestRunner", lambda: TargetedRunner(test_file)):
with self._patched_runner_for(test_file):
result = run_sanity.run_pytests()
assert result["failed"] == 1
@@ -701,3 +712,32 @@ class TestFailedTestReportIntegration:
" - TestDemo : test_broken_assert\n"
" - assert 'Test Document' == 'Test Documen'"
) in result["stdout"]
def test_real_error_shows_function_and_message(self, tmp_path):
"""Test a real pytest error (fixture failure) nests under the file.
A test that errors in a fixture is counted as failed in the file
listing; it must also show its name and error message instead of
vanishing from the report.
"""
test_file = tmp_path / "test_integration_error.py"
test_file.write_text(
"import pytest\n"
"\n"
"@pytest.fixture\n"
"def broken_fixture():\n"
" raise RuntimeError('boom')\n"
"\n"
"def test_needs_fixture(broken_fixture):\n"
" assert True\n",
encoding="utf-8",
)
with self._patched_runner_for(test_file):
result = run_sanity.run_pytests()
assert result["failed"] == 1
assert (" - test_integration_error.py 0/1 passed, 1 failed\n"
" - test_needs_fixture\n"
" - RuntimeError: boom"
) in result["stdout"]
+52
View File
@@ -67,6 +67,19 @@ class TestParsePytestOutput:
assert result["failed"] == 0
assert result["skipped"] == 1
def test_parse_output_with_errors(self, test_runner):
"""Test errored tests count as failures in the summary counts."""
output = "============================= 2 passed, 1 error in 0.10s ======================"
result = test_runner._parse_pytest_output(output)
assert result["passed"] == 2
assert result["failed"] == 1
def test_parse_output_with_plural_errors(self, test_runner):
"""Test multiple errored tests count as failures."""
output = "============================= 2 errors in 0.10s ==============================="
result = test_runner._parse_pytest_output(output)
assert result["failed"] == 2
def test_parse_output_with_neither(self, test_runner):
"""Test parsing output with neither passed nor failed."""
output = "No tests collected"
@@ -189,6 +202,45 @@ class TestRunPytest:
assert failures[0]["location"] is None
assert failures[0]["assert"] is None
def test_parse_error_tests_output(self, test_runner):
"""Test errored tests (fixture errors) are reported with id and message."""
output = (
"=========================== short test summary info ============================\n"
"ERROR src/tests/test_a.py::TestX::test_one - RuntimeError: boom\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 1
assert failures[0]["id"] == "src/tests/test_a.py::TestX::test_one"
assert failures[0]["message"] == "RuntimeError: boom"
assert failures[0]["location"] is None
assert failures[0]["assert"] is None
def test_parse_mixed_failed_and_error_tests(self, test_runner):
"""Test FAILED entries keep traceback details while ERROR entries do not."""
output = (
"=================================== FAILURES ===================================\n"
"_______________________ TestX.test_that_fails ________________________________\n"
"\n"
"> assert 1 == 2\n"
"E assert 1 == 2\n"
"\n"
"src/tests/test_a.py:5: AssertionError\n"
"=========================== short test summary info ============================\n"
"FAILED src/tests/test_a.py::TestX::test_that_fails - assert 1 == 2\n"
"ERROR src/tests/test_b.py::TestY::test_broken_fixture - RuntimeError: boom\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 2
failed = failures[0]
assert failed["id"] == "src/tests/test_a.py::TestX::test_that_fails"
assert failed["location"] == "src/tests/test_a.py:5"
assert failed["assert"] == "assert 1 == 2"
errored = failures[1]
assert errored["id"] == "src/tests/test_b.py::TestY::test_broken_fixture"
assert errored["message"] == "RuntimeError: boom"
assert errored["location"] is None
assert errored["assert"] is None
@patch('run_tests.subprocess.run')
def test_run_pytest_includes_failed_tests(self, mock_run, test_runner):
"""Test run_pytest returns the failed test details."""
+1 -1
View File
@@ -112,7 +112,7 @@ block a correct install or make the docs contradict the code.
- [ ] it signifies that is the root of the llm-wiki
- [x] When running sanity, make sure that markdownlint tests also include
outer dir of the src/ because there are markdown files there
- [x] in `run_sanity.py`, show the failing test function and assert ex.
- [ ] in `run_sanity.py`, show the failing test function and assert ex.
```text
- test_md_frontmatter.py 43/44 passed, 1 failed