run_sanity tests left failing dues to AI gaslighting
This commit is contained in:
@@ -84,11 +84,6 @@ def run_pytests():
|
||||
if counts["skipped"]:
|
||||
detail += f", {counts['skipped']} skipped"
|
||||
lines.append(f" - {file_name:<{name_width}} {detail}")
|
||||
for failure in failures_by_file.get(file_name, []):
|
||||
parts = failure["id"].split("::")
|
||||
test_name = " : ".join(parts[1:]) if len(parts) > 1 else failure["id"]
|
||||
lines.append(f" - {test_name}")
|
||||
lines.append(f" - {failure.get('assert') or failure['message']}")
|
||||
|
||||
failed_tests = result.get("failed_tests", [])
|
||||
if failed_tests:
|
||||
|
||||
+23
-8
@@ -35,7 +35,9 @@ class TestRunner:
|
||||
|
||||
The counts come from the final summary line, so the last match
|
||||
wins: assertion diffs printed by failing tests can contain
|
||||
earlier `N passed` strings that must not be counted.
|
||||
earlier `N passed` strings that must not be counted. Errored
|
||||
tests (fixture or collection errors) count as failures so the
|
||||
totals match the per-file listing.
|
||||
"""
|
||||
def last_count(pattern: str) -> int:
|
||||
matches = re.findall(pattern, output)
|
||||
@@ -43,7 +45,7 @@ class TestRunner:
|
||||
|
||||
return {
|
||||
"passed": last_count(r'(\d+) passed'),
|
||||
"failed": last_count(r'(\d+) failed'),
|
||||
"failed": last_count(r'(\d+) failed') + last_count(r'(\d+) errors?'),
|
||||
"skipped": last_count(r'(\d+) skipped'),
|
||||
}
|
||||
|
||||
@@ -80,12 +82,14 @@ class TestRunner:
|
||||
return file_coverage
|
||||
|
||||
def _parse_failed_tests_output(self, output: str) -> list[dict]:
|
||||
"""Parse the pytest output into failed test details.
|
||||
"""Parse the pytest output into failed and errored test details.
|
||||
|
||||
Combines the short summary (test id and exception line) with the
|
||||
FAILURES traceback section (assert location and explanation) so
|
||||
reports show what failed, where, and why. Failures are paired
|
||||
with their traceback blocks by order.
|
||||
reports show what failed, where, and why. Errored tests (fixture
|
||||
or collection errors, reported as `ERROR` lines) are included
|
||||
with their id and message so they do not vanish from the report.
|
||||
Failures are paired with their traceback blocks by order.
|
||||
|
||||
Args:
|
||||
output: The pytest output text.
|
||||
@@ -96,21 +100,32 @@ class TestRunner:
|
||||
the failing assert), and the `assert` explanation.
|
||||
"""
|
||||
failures: list[dict] = []
|
||||
pattern = re.compile(r"^FAILED (\S+) - (.+)$", re.MULTILINE)
|
||||
for match in pattern.finditer(output):
|
||||
errors: list[dict] = []
|
||||
failed_pattern = re.compile(r"^FAILED (\S+) - (.+)$", re.MULTILINE)
|
||||
error_pattern = re.compile(r"^ERROR (\S+) - (.+)$", re.MULTILINE)
|
||||
for match in failed_pattern.finditer(output):
|
||||
failures.append({
|
||||
"id": match.group(1),
|
||||
"message": match.group(2),
|
||||
"location": None,
|
||||
"assert": None,
|
||||
})
|
||||
for match in error_pattern.finditer(output):
|
||||
errors.append({
|
||||
"id": match.group(1),
|
||||
"message": match.group(2),
|
||||
"location": None,
|
||||
"assert": None,
|
||||
})
|
||||
|
||||
# traceback details only exist for FAILURES section entries, so
|
||||
# they pair with the FAILED entries, never with the errored ones
|
||||
details = self._parse_failures_section(output)
|
||||
if len(details) == len(failures):
|
||||
for failure, detail in zip(failures, details):
|
||||
failure["location"] = detail["location"]
|
||||
failure["assert"] = detail["assert"]
|
||||
return failures
|
||||
return failures + errors
|
||||
|
||||
def _parse_failures_section(self, output: str) -> list[dict]:
|
||||
"""Parse the FAILURES traceback section into per-test details.
|
||||
|
||||
@@ -663,7 +663,31 @@ class TestVerdict:
|
||||
|
||||
|
||||
class TestFailedTestReportIntegration:
|
||||
"""End-to-end check that a real failing test reaches the report."""
|
||||
"""End-to-end checks that real failures reach the report."""
|
||||
|
||||
@staticmethod
|
||||
def _patched_runner_for(test_file):
|
||||
"""Patch run_sanity.TestRunner with a real runner for a test file.
|
||||
|
||||
The runner runs pytest in a real subprocess (no mocks, no coverage)
|
||||
against the given file so the whole chain is exercised: real pytest
|
||||
output, the failure parsing, and the report formatting in
|
||||
run_pytests.
|
||||
"""
|
||||
class TargetedRunner(TestRunner):
|
||||
"""Real TestRunner redirected to the given test file."""
|
||||
|
||||
def __init__(self, target):
|
||||
super().__init__()
|
||||
self.target = target
|
||||
|
||||
def run_pytest(self, test_path, verbose=False, capture_output=False,
|
||||
with_coverage=True, ignore=None):
|
||||
return super().run_pytest(
|
||||
self.target, verbose, capture_output, with_coverage=False
|
||||
)
|
||||
|
||||
return patch("run_sanity.TestRunner", lambda: TargetedRunner(test_file))
|
||||
|
||||
def test_real_failure_shows_function_and_assert(self, tmp_path):
|
||||
"""Test a real pytest failure nests its name and assert under the file.
|
||||
@@ -680,20 +704,7 @@ class TestFailedTestReportIntegration:
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
class TargetedRunner(TestRunner):
|
||||
"""Real TestRunner redirected to the temporary test file."""
|
||||
|
||||
def __init__(self, target):
|
||||
super().__init__()
|
||||
self.target = target
|
||||
|
||||
def run_pytest(self, test_path, verbose=False, capture_output=False,
|
||||
with_coverage=True, ignore=None):
|
||||
return super().run_pytest(
|
||||
self.target, verbose, capture_output, with_coverage=False
|
||||
)
|
||||
|
||||
with patch("run_sanity.TestRunner", lambda: TargetedRunner(test_file)):
|
||||
with self._patched_runner_for(test_file):
|
||||
result = run_sanity.run_pytests()
|
||||
|
||||
assert result["failed"] == 1
|
||||
@@ -701,3 +712,32 @@ class TestFailedTestReportIntegration:
|
||||
" - TestDemo : test_broken_assert\n"
|
||||
" - assert 'Test Document' == 'Test Documen'"
|
||||
) in result["stdout"]
|
||||
|
||||
def test_real_error_shows_function_and_message(self, tmp_path):
|
||||
"""Test a real pytest error (fixture failure) nests under the file.
|
||||
|
||||
A test that errors in a fixture is counted as failed in the file
|
||||
listing; it must also show its name and error message instead of
|
||||
vanishing from the report.
|
||||
"""
|
||||
test_file = tmp_path / "test_integration_error.py"
|
||||
test_file.write_text(
|
||||
"import pytest\n"
|
||||
"\n"
|
||||
"@pytest.fixture\n"
|
||||
"def broken_fixture():\n"
|
||||
" raise RuntimeError('boom')\n"
|
||||
"\n"
|
||||
"def test_needs_fixture(broken_fixture):\n"
|
||||
" assert True\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with self._patched_runner_for(test_file):
|
||||
result = run_sanity.run_pytests()
|
||||
|
||||
assert result["failed"] == 1
|
||||
assert (" - test_integration_error.py 0/1 passed, 1 failed\n"
|
||||
" - test_needs_fixture\n"
|
||||
" - RuntimeError: boom"
|
||||
) in result["stdout"]
|
||||
|
||||
@@ -67,6 +67,19 @@ class TestParsePytestOutput:
|
||||
assert result["failed"] == 0
|
||||
assert result["skipped"] == 1
|
||||
|
||||
def test_parse_output_with_errors(self, test_runner):
|
||||
"""Test errored tests count as failures in the summary counts."""
|
||||
output = "============================= 2 passed, 1 error in 0.10s ======================"
|
||||
result = test_runner._parse_pytest_output(output)
|
||||
assert result["passed"] == 2
|
||||
assert result["failed"] == 1
|
||||
|
||||
def test_parse_output_with_plural_errors(self, test_runner):
|
||||
"""Test multiple errored tests count as failures."""
|
||||
output = "============================= 2 errors in 0.10s ==============================="
|
||||
result = test_runner._parse_pytest_output(output)
|
||||
assert result["failed"] == 2
|
||||
|
||||
def test_parse_output_with_neither(self, test_runner):
|
||||
"""Test parsing output with neither passed nor failed."""
|
||||
output = "No tests collected"
|
||||
@@ -189,6 +202,45 @@ class TestRunPytest:
|
||||
assert failures[0]["location"] is None
|
||||
assert failures[0]["assert"] is None
|
||||
|
||||
def test_parse_error_tests_output(self, test_runner):
|
||||
"""Test errored tests (fixture errors) are reported with id and message."""
|
||||
output = (
|
||||
"=========================== short test summary info ============================\n"
|
||||
"ERROR src/tests/test_a.py::TestX::test_one - RuntimeError: boom\n"
|
||||
)
|
||||
failures = test_runner._parse_failed_tests_output(output)
|
||||
assert len(failures) == 1
|
||||
assert failures[0]["id"] == "src/tests/test_a.py::TestX::test_one"
|
||||
assert failures[0]["message"] == "RuntimeError: boom"
|
||||
assert failures[0]["location"] is None
|
||||
assert failures[0]["assert"] is None
|
||||
|
||||
def test_parse_mixed_failed_and_error_tests(self, test_runner):
|
||||
"""Test FAILED entries keep traceback details while ERROR entries do not."""
|
||||
output = (
|
||||
"=================================== FAILURES ===================================\n"
|
||||
"_______________________ TestX.test_that_fails ________________________________\n"
|
||||
"\n"
|
||||
"> assert 1 == 2\n"
|
||||
"E assert 1 == 2\n"
|
||||
"\n"
|
||||
"src/tests/test_a.py:5: AssertionError\n"
|
||||
"=========================== short test summary info ============================\n"
|
||||
"FAILED src/tests/test_a.py::TestX::test_that_fails - assert 1 == 2\n"
|
||||
"ERROR src/tests/test_b.py::TestY::test_broken_fixture - RuntimeError: boom\n"
|
||||
)
|
||||
failures = test_runner._parse_failed_tests_output(output)
|
||||
assert len(failures) == 2
|
||||
failed = failures[0]
|
||||
assert failed["id"] == "src/tests/test_a.py::TestX::test_that_fails"
|
||||
assert failed["location"] == "src/tests/test_a.py:5"
|
||||
assert failed["assert"] == "assert 1 == 2"
|
||||
errored = failures[1]
|
||||
assert errored["id"] == "src/tests/test_b.py::TestY::test_broken_fixture"
|
||||
assert errored["message"] == "RuntimeError: boom"
|
||||
assert errored["location"] is None
|
||||
assert errored["assert"] is None
|
||||
|
||||
@patch('run_tests.subprocess.run')
|
||||
def test_run_pytest_includes_failed_tests(self, mock_run, test_runner):
|
||||
"""Test run_pytest returns the failed test details."""
|
||||
|
||||
@@ -112,7 +112,7 @@ block a correct install or make the docs contradict the code.
|
||||
- [ ] it signifies that is the root of the llm-wiki
|
||||
- [x] When running sanity, make sure that markdownlint tests also include
|
||||
outer dir of the src/ because there are markdown files there
|
||||
- [x] in `run_sanity.py`, show the failing test function and assert ex.
|
||||
- [ ] in `run_sanity.py`, show the failing test function and assert ex.
|
||||
|
||||
```text
|
||||
- test_md_frontmatter.py 43/44 passed, 1 failed
|
||||
|
||||
Reference in New Issue
Block a user