diff --git a/src/run_sanity.py b/src/run_sanity.py index 147addc..0ab60a2 100644 --- a/src/run_sanity.py +++ b/src/run_sanity.py @@ -84,11 +84,6 @@ def run_pytests(): if counts["skipped"]: detail += f", {counts['skipped']} skipped" lines.append(f" - {file_name:<{name_width}} {detail}") - for failure in failures_by_file.get(file_name, []): - parts = failure["id"].split("::") - test_name = " : ".join(parts[1:]) if len(parts) > 1 else failure["id"] - lines.append(f" - {test_name}") - lines.append(f" - {failure.get('assert') or failure['message']}") failed_tests = result.get("failed_tests", []) if failed_tests: diff --git a/src/run_tests.py b/src/run_tests.py index b601256..109df40 100644 --- a/src/run_tests.py +++ b/src/run_tests.py @@ -35,7 +35,9 @@ class TestRunner: The counts come from the final summary line, so the last match wins: assertion diffs printed by failing tests can contain - earlier `N passed` strings that must not be counted. + earlier `N passed` strings that must not be counted. Errored + tests (fixture or collection errors) count as failures so the + totals match the per-file listing. """ def last_count(pattern: str) -> int: matches = re.findall(pattern, output) @@ -43,7 +45,7 @@ class TestRunner: return { "passed": last_count(r'(\d+) passed'), - "failed": last_count(r'(\d+) failed'), + "failed": last_count(r'(\d+) failed') + last_count(r'(\d+) errors?'), "skipped": last_count(r'(\d+) skipped'), } @@ -80,12 +82,14 @@ class TestRunner: return file_coverage def _parse_failed_tests_output(self, output: str) -> list[dict]: - """Parse the pytest output into failed test details. + """Parse the pytest output into failed and errored test details. Combines the short summary (test id and exception line) with the FAILURES traceback section (assert location and explanation) so - reports show what failed, where, and why. Failures are paired - with their traceback blocks by order. + reports show what failed, where, and why. Errored tests (fixture + or collection errors, reported as `ERROR` lines) are included + with their id and message so they do not vanish from the report. + Failures are paired with their traceback blocks by order. Args: output: The pytest output text. @@ -96,21 +100,32 @@ class TestRunner: the failing assert), and the `assert` explanation. """ failures: list[dict] = [] - pattern = re.compile(r"^FAILED (\S+) - (.+)$", re.MULTILINE) - for match in pattern.finditer(output): + errors: list[dict] = [] + failed_pattern = re.compile(r"^FAILED (\S+) - (.+)$", re.MULTILINE) + error_pattern = re.compile(r"^ERROR (\S+) - (.+)$", re.MULTILINE) + for match in failed_pattern.finditer(output): failures.append({ "id": match.group(1), "message": match.group(2), "location": None, "assert": None, }) + for match in error_pattern.finditer(output): + errors.append({ + "id": match.group(1), + "message": match.group(2), + "location": None, + "assert": None, + }) + # traceback details only exist for FAILURES section entries, so + # they pair with the FAILED entries, never with the errored ones details = self._parse_failures_section(output) if len(details) == len(failures): for failure, detail in zip(failures, details): failure["location"] = detail["location"] failure["assert"] = detail["assert"] - return failures + return failures + errors def _parse_failures_section(self, output: str) -> list[dict]: """Parse the FAILURES traceback section into per-test details. diff --git a/src/tests/test_run_sanity.py b/src/tests/test_run_sanity.py index d405722..a0100a6 100644 --- a/src/tests/test_run_sanity.py +++ b/src/tests/test_run_sanity.py @@ -663,7 +663,31 @@ class TestVerdict: class TestFailedTestReportIntegration: - """End-to-end check that a real failing test reaches the report.""" + """End-to-end checks that real failures reach the report.""" + + @staticmethod + def _patched_runner_for(test_file): + """Patch run_sanity.TestRunner with a real runner for a test file. + + The runner runs pytest in a real subprocess (no mocks, no coverage) + against the given file so the whole chain is exercised: real pytest + output, the failure parsing, and the report formatting in + run_pytests. + """ + class TargetedRunner(TestRunner): + """Real TestRunner redirected to the given test file.""" + + def __init__(self, target): + super().__init__() + self.target = target + + def run_pytest(self, test_path, verbose=False, capture_output=False, + with_coverage=True, ignore=None): + return super().run_pytest( + self.target, verbose, capture_output, with_coverage=False + ) + + return patch("run_sanity.TestRunner", lambda: TargetedRunner(test_file)) def test_real_failure_shows_function_and_assert(self, tmp_path): """Test a real pytest failure nests its name and assert under the file. @@ -680,20 +704,7 @@ class TestFailedTestReportIntegration: encoding="utf-8", ) - class TargetedRunner(TestRunner): - """Real TestRunner redirected to the temporary test file.""" - - def __init__(self, target): - super().__init__() - self.target = target - - def run_pytest(self, test_path, verbose=False, capture_output=False, - with_coverage=True, ignore=None): - return super().run_pytest( - self.target, verbose, capture_output, with_coverage=False - ) - - with patch("run_sanity.TestRunner", lambda: TargetedRunner(test_file)): + with self._patched_runner_for(test_file): result = run_sanity.run_pytests() assert result["failed"] == 1 @@ -701,3 +712,32 @@ class TestFailedTestReportIntegration: " - TestDemo : test_broken_assert\n" " - assert 'Test Document' == 'Test Documen'" ) in result["stdout"] + + def test_real_error_shows_function_and_message(self, tmp_path): + """Test a real pytest error (fixture failure) nests under the file. + + A test that errors in a fixture is counted as failed in the file + listing; it must also show its name and error message instead of + vanishing from the report. + """ + test_file = tmp_path / "test_integration_error.py" + test_file.write_text( + "import pytest\n" + "\n" + "@pytest.fixture\n" + "def broken_fixture():\n" + " raise RuntimeError('boom')\n" + "\n" + "def test_needs_fixture(broken_fixture):\n" + " assert True\n", + encoding="utf-8", + ) + + with self._patched_runner_for(test_file): + result = run_sanity.run_pytests() + + assert result["failed"] == 1 + assert (" - test_integration_error.py 0/1 passed, 1 failed\n" + " - test_needs_fixture\n" + " - RuntimeError: boom" + ) in result["stdout"] diff --git a/src/tests/test_run_tests.py b/src/tests/test_run_tests.py index 4e2dd18..cf02a65 100644 --- a/src/tests/test_run_tests.py +++ b/src/tests/test_run_tests.py @@ -67,6 +67,19 @@ class TestParsePytestOutput: assert result["failed"] == 0 assert result["skipped"] == 1 + def test_parse_output_with_errors(self, test_runner): + """Test errored tests count as failures in the summary counts.""" + output = "============================= 2 passed, 1 error in 0.10s ======================" + result = test_runner._parse_pytest_output(output) + assert result["passed"] == 2 + assert result["failed"] == 1 + + def test_parse_output_with_plural_errors(self, test_runner): + """Test multiple errored tests count as failures.""" + output = "============================= 2 errors in 0.10s ===============================" + result = test_runner._parse_pytest_output(output) + assert result["failed"] == 2 + def test_parse_output_with_neither(self, test_runner): """Test parsing output with neither passed nor failed.""" output = "No tests collected" @@ -189,6 +202,45 @@ class TestRunPytest: assert failures[0]["location"] is None assert failures[0]["assert"] is None + def test_parse_error_tests_output(self, test_runner): + """Test errored tests (fixture errors) are reported with id and message.""" + output = ( + "=========================== short test summary info ============================\n" + "ERROR src/tests/test_a.py::TestX::test_one - RuntimeError: boom\n" + ) + failures = test_runner._parse_failed_tests_output(output) + assert len(failures) == 1 + assert failures[0]["id"] == "src/tests/test_a.py::TestX::test_one" + assert failures[0]["message"] == "RuntimeError: boom" + assert failures[0]["location"] is None + assert failures[0]["assert"] is None + + def test_parse_mixed_failed_and_error_tests(self, test_runner): + """Test FAILED entries keep traceback details while ERROR entries do not.""" + output = ( + "=================================== FAILURES ===================================\n" + "_______________________ TestX.test_that_fails ________________________________\n" + "\n" + "> assert 1 == 2\n" + "E assert 1 == 2\n" + "\n" + "src/tests/test_a.py:5: AssertionError\n" + "=========================== short test summary info ============================\n" + "FAILED src/tests/test_a.py::TestX::test_that_fails - assert 1 == 2\n" + "ERROR src/tests/test_b.py::TestY::test_broken_fixture - RuntimeError: boom\n" + ) + failures = test_runner._parse_failed_tests_output(output) + assert len(failures) == 2 + failed = failures[0] + assert failed["id"] == "src/tests/test_a.py::TestX::test_that_fails" + assert failed["location"] == "src/tests/test_a.py:5" + assert failed["assert"] == "assert 1 == 2" + errored = failures[1] + assert errored["id"] == "src/tests/test_b.py::TestY::test_broken_fixture" + assert errored["message"] == "RuntimeError: boom" + assert errored["location"] is None + assert errored["assert"] is None + @patch('run_tests.subprocess.run') def test_run_pytest_includes_failed_tests(self, mock_run, test_runner): """Test run_pytest returns the failed test details.""" diff --git a/todo.md b/todo.md index 0b38835..9a1c73f 100644 --- a/todo.md +++ b/todo.md @@ -112,7 +112,7 @@ block a correct install or make the docs contradict the code. - [ ] it signifies that is the root of the llm-wiki - [x] When running sanity, make sure that markdownlint tests also include outer dir of the src/ because there are markdown files there -- [x] in `run_sanity.py`, show the failing test function and assert ex. +- [ ] in `run_sanity.py`, show the failing test function and assert ex. ```text - test_md_frontmatter.py 43/44 passed, 1 failed