Files
fiona-assistant/tests/test_run_tests.py
T
2026-09-29 16:01:16 +02:00

1091 lines
47 KiB
Python

"""Tests for the run_tests.py TestRunner module."""
import logging
import pathlib
from unittest.mock import MagicMock, patch
import pytest
import fiona.run_tests as run_tests
@pytest.fixture
def test_runner():
"""Fixture providing a TestRunner instance."""
return run_tests.TestRunner()
class TestTestRunnerInit:
"""Tests for TestRunner.__init__."""
def test_init_default_base_dir(self):
"""Test that TestRunner initializes with default base_dir of '.'."""
runner = run_tests.TestRunner()
assert runner.base_dir == "."
def test_init_custom_base_dir(self):
"""Test that TestRunner accepts a custom base_dir."""
runner = run_tests.TestRunner(base_dir="/custom/path")
assert runner.base_dir == "/custom/path"
def test_init_counters_zero(self):
"""Test that TestRunner initializes counters to zero."""
runner = run_tests.TestRunner()
assert runner.total_collected == 0
assert runner.total_passed == 0
assert runner.total_failed == 0
assert runner.test_results == {}
class TestParsePytestOutput:
"""Tests for TestRunner._parse_pytest_output."""
def test_parse_output_with_passed(self, test_runner):
"""Test parsing output with passed tests."""
output = "============================= 5 passed in 0.10s ==============================="
result = test_runner._parse_pytest_output(output)
assert result["passed"] == 5
assert result["failed"] == 0
def test_parse_output_with_failed(self, test_runner):
"""Test parsing output with failed tests."""
output = "============================= 2 failed in 0.10s ==============================="
result = test_runner._parse_pytest_output(output)
assert result["passed"] == 0
assert result["failed"] == 2
def test_parse_output_with_both(self, test_runner):
"""Test parsing output with both passed and failed tests."""
output = "============================= 3 passed, 2 failed in 0.10s ==============================="
result = test_runner._parse_pytest_output(output)
assert result["passed"] == 3
assert result["failed"] == 2
def test_parse_output_with_skipped(self, test_runner):
"""Test parsing output with skipped tests."""
output = "============================= 2 passed, 1 skipped in 0.10s ==============================="
result = test_runner._parse_pytest_output(output)
assert result["passed"] == 2
assert result["failed"] == 0
assert result["skipped"] == 1
def test_parse_output_with_errors(self, test_runner):
"""Test errored tests count as failures in the summary counts."""
output = "============================= 2 passed, 1 error in 0.10s ======================"
result = test_runner._parse_pytest_output(output)
assert result["passed"] == 2
assert result["failed"] == 1
def test_parse_output_with_plural_errors(self, test_runner):
"""Test multiple errored tests count as failures."""
output = "============================= 2 errors in 0.10s ==============================="
result = test_runner._parse_pytest_output(output)
assert result["failed"] == 2
def test_parse_output_with_neither(self, test_runner):
"""Test parsing output with neither passed nor failed."""
output = "No tests collected"
result = test_runner._parse_pytest_output(output)
assert result["passed"] == 0
assert result["failed"] == 0
def test_parse_empty_output(self, test_runner):
"""Test parsing empty output."""
result = test_runner._parse_pytest_output("")
assert result["passed"] == 0
assert result["failed"] == 0
def test_parse_output_takes_last_match(self, test_runner):
"""Test counts come from the final summary, not assertion diffs.
Failing tests print assertion diffs that can contain strings like
`2/2 passed` before the summary line; only the summary counts.
"""
output = (
"AssertionError: assert 'test_a.py 2/2 passed' in 'Pytest Results'\n"
"FAILED src/tests/test_run_sanity.py::TestX::test_y - AssertionError\n"
"============================= 214 passed, 5 failed in 0.10s ==============\n"
)
result = test_runner._parse_pytest_output(output)
assert result["passed"] == 214
assert result["failed"] == 5
assert result["skipped"] == 0
class TestRunPytest:
"""Tests for TestRunner.run_pytest."""
def test_parse_test_files_output(self, test_runner):
"""Test parsing per-file results from pytest verbose output."""
output = (
"src/tests/test_a.py::TestX::test_one PASSED [ 10%]\n"
"src/tests/test_a.py::TestX::test_two SKIPPED [ 20%]\n"
"src/tests/test_b.py::TestY::test_three FAILED [ 30%]\n"
"============================= 1 passed, 1 failed, 1 skipped in 0.10s ====\n"
"FAILED src/tests/test_b.py::TestY::test_three - AssertionError\n"
)
results = test_runner._parse_test_files_output(output)
assert results["test_a.py"] == {
"passed": 1, "failed": 0, "skipped": 1, "total": 2
}
assert results["test_b.py"] == {
"passed": 0, "failed": 1, "skipped": 0, "total": 1
}
def test_parse_test_files_output_empty(self, test_runner):
"""Test parsing per-file results from output without test lines."""
assert test_runner._parse_test_files_output("") == {}
def test_parse_coverage_files_output(self, test_runner):
"""Test parsing the coverage table into per-file percentages."""
output = (
"Name Stmts Miss Cover\n"
"-----------------------------------------------------\n"
"src/__init__.py 1 0 100%\n"
"src/mdoctor/mdoctor.py 149 2 99%\n"
"-----------------------------------------------------\n"
"TOTAL 798 27 97%\n"
)
coverage = test_runner._parse_coverage_files_output(output)
assert coverage["src/__init__.py"] == 100.0
assert coverage["src/mdoctor/mdoctor.py"] == 99.0
assert "TOTAL" not in coverage
def test_parse_coverage_files_output_empty(self, test_runner):
"""Test parsing per-file coverage from output without a table."""
assert test_runner._parse_coverage_files_output("") == {}
def test_parse_failed_tests_output(self, test_runner):
"""Test parsing failed test ids and messages from the summary."""
output = (
"=========================== short test summary info ============================\n"
"FAILED src/tests/test_a.py::TestX::test_one - AssertionError: assert 1 == 2\n"
"FAILED src/tests/test_b.py::TestY::test_two - TypeError: bad call\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 2
assert failures[0]["id"] == "src/tests/test_a.py::TestX::test_one"
assert failures[0]["message"] == "AssertionError: assert 1 == 2"
assert failures[1]["id"] == "src/tests/test_b.py::TestY::test_two"
def test_parse_failed_tests_output_empty(self, test_runner):
"""Test parsing failed tests from output without failures."""
assert test_runner._parse_failed_tests_output("") == []
def test_parse_failed_tests_with_traceback_details(self, test_runner):
"""Test failed tests carry the assert location and explanation."""
output = (
"=================================== FAILURES ===================================\n"
"_______________________ TestX.test_that_fails ________________________________\n"
"\n"
" def test_that_fails():\n"
"> assert 1 == 2\n"
"E assert 1 == 2\n"
"\n"
"src/tests/test_a.py:5: AssertionError\n"
"=========================== short test summary info ============================\n"
"FAILED src/tests/test_a.py::TestX::test_that_fails - assert 1 == 2\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 1
assert failures[0]["id"] == "src/tests/test_a.py::TestX::test_that_fails"
assert failures[0]["location"] == "src/tests/test_a.py:5"
assert failures[0]["assert"] == "assert 1 == 2"
def test_parse_failed_tests_without_traceback_keeps_message(self, test_runner):
"""Test failures without a FAILURES section keep id and message only."""
output = (
"=========================== short test summary info ============================\n"
"FAILED src/tests/test_a.py::TestX::test_one - AssertionError: assert 1 == 2\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert failures[0]["id"] == "src/tests/test_a.py::TestX::test_one"
assert failures[0]["message"] == "AssertionError: assert 1 == 2"
assert failures[0]["location"] is None
assert failures[0]["assert"] is None
def test_parse_error_tests_output(self, test_runner):
"""Test errored tests (fixture errors) are reported with id and message."""
output = (
"=========================== short test summary info ============================\n"
"ERROR src/tests/test_a.py::TestX::test_one - RuntimeError: boom\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 1
assert failures[0]["id"] == "src/tests/test_a.py::TestX::test_one"
assert failures[0]["message"] == "RuntimeError: boom"
assert failures[0]["location"] is None
assert failures[0]["assert"] is None
def test_parse_mixed_failed_and_error_tests(self, test_runner):
"""Test FAILED entries keep traceback details while ERROR entries do not."""
output = (
"=================================== FAILURES ===================================\n"
"_______________________ TestX.test_that_fails ________________________________\n"
"\n"
"> assert 1 == 2\n"
"E assert 1 == 2\n"
"\n"
"src/tests/test_a.py:5: AssertionError\n"
"=========================== short test summary info ============================\n"
"FAILED src/tests/test_a.py::TestX::test_that_fails - assert 1 == 2\n"
"ERROR src/tests/test_b.py::TestY::test_broken_fixture - RuntimeError: boom\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 2
failed = failures[0]
assert failed["id"] == "src/tests/test_a.py::TestX::test_that_fails"
assert failed["location"] == "src/tests/test_a.py:5"
assert failed["assert"] == "assert 1 == 2"
errored = failures[1]
assert errored["id"] == "src/tests/test_b.py::TestY::test_broken_fixture"
assert errored["message"] == "RuntimeError: boom"
assert errored["location"] is None
assert errored["assert"] is None
def test_parse_failed_tests_fallback_to_verbose_lines(self, test_runner):
"""Test failures are still reported when the short summary is missing."""
output = (
"src/tests/test_a.py::TestX::test_one PASSED [ 50%]\n"
"src/tests/test_b.py::TestY::test_two FAILED [100%]\n"
"========================= 1 passed, 1 failed in 0.10s ========================\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 1
assert failures[0]["id"] == "src/tests/test_b.py::TestY::test_two"
assert failures[0]["message"] == "test failed (no short summary in pytest output)"
assert failures[0]["location"] is None
assert failures[0]["assert"] is None
def test_parse_error_tests_fallback_to_verbose_lines(self, test_runner):
"""Test errored tests are reported from verbose lines without summary."""
output = (
"src/tests/test_a.py::test_needs_fixture ERROR [100%]\n"
"============================== 1 error in 0.10s ============================\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 1
assert failures[0]["id"] == "src/tests/test_a.py::test_needs_fixture"
assert failures[0]["message"] == "test errored (no short summary in pytest output)"
assert failures[0]["location"] is None
assert failures[0]["assert"] is None
def test_parse_failed_tests_fallback_no_duplicates(self, test_runner):
"""Test the verbose fallback does not duplicate short-summary entries."""
output = (
"src/tests/test_b.py::TestY::test_two FAILED [100%]\n"
"FAILED src/tests/test_b.py::TestY::test_two - AssertionError: assert 1 == 2\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert len(failures) == 1
assert failures[0]["message"] == "AssertionError: assert 1 == 2"
def test_parse_junit_failures_missing_file(self, test_runner, tmp_path):
"""Test junit parsing returns no entries without a report file."""
assert test_runner._parse_junit_failures(tmp_path / "absent.xml") == []
@staticmethod
def _write_junit_report(junit_path, failure=True, error=True):
"""Write a small xunit1-style JUnit report for testing.
Args:
junit_path: Path to write the XML report to.
failure: Include a failed assert testcase.
error: Include an errored (fixture) testcase.
"""
cases = [
('<testcase classname="src.tests.test_a.TestX" name="test_ok"'
' file="src/tests/test_a.py" line="5" time="0.001"/>'),
]
if failure:
cases.append(
'<testcase classname="src.tests.test_a.TestX" name="test_one"'
' file="src/tests/test_a.py" line="10" time="0.001">'
'<failure message="AssertionError: assert 1 == 2&#10; extra diff"'
' type="assert">traceback text</failure></testcase>'
)
if error:
cases.append(
'<testcase classname="src.tests.test_b" name="test_needs_fixture"'
' file="src/tests/test_b.py" line="3" time="0.001">'
'<error message="failed on setup with &quot;RuntimeError: boom&quot;"'
' type="test">traceback text</error></testcase>'
)
xml = (
'<?xml version="1.0" encoding="utf-8"?>\n'
'<testsuites><testsuite name="pytest" errors="1" failures="1"'
' skipped="0" tests="3" time="0.1">\n'
+ "\n".join(cases)
+ "\n</testsuite></testsuites>\n"
)
pathlib.Path(junit_path).write_text(xml, encoding="utf-8")
def test_run_pytest_uses_junit_details_when_summary_missing(self, mocker,
test_runner):
"""Test junit report recovers failure details without a short summary.
This mimics environments where pytest's short summary (and
tracebacks) are missing from the captured output: the JUnit XML
report is the environment-independent source for the failing
test id, location, and assert message.
"""
def fake_run(cmd, **kwargs):
self._write_junit_report(cmd[cmd.index("--junitxml") + 1], error=False)
return MagicMock(
returncode=1,
stdout=(
"src/tests/test_a.py::TestX::test_one FAILED [ 50%]\n"
"========================= 1 failed in 0.10s ========================\n"
),
stderr="",
)
mocker.patch("fiona.run_tests.subprocess.run", side_effect=fake_run)
result = test_runner.run_pytest("src/tests", verbose=True,
with_coverage=False, capture_output=True)
assert len(result["failed_tests"]) == 1
failure = result["failed_tests"][0]
assert failure["id"] == "src/tests/test_a.py::TestX::test_one"
assert failure["message"] == "AssertionError: assert 1 == 2"
assert failure["location"] == "src/tests/test_a.py:10"
assert failure["assert"] == "assert 1 == 2"
def test_run_pytest_junit_reports_error_entries(self, mocker, test_runner):
"""Test junit errored testcases report the unwrapped error message.
pytest wraps fixture errors as `failed on setup with "..."` in
the junit report; the runner unwraps them so the displayed
message matches the short summary form regardless of which
source provided the details.
"""
def fake_run(cmd, **kwargs):
self._write_junit_report(cmd[cmd.index("--junitxml") + 1], failure=False)
return MagicMock(
returncode=1,
stdout="========================= 1 error in 0.10s =========================\n",
stderr="",
)
mocker.patch("fiona.run_tests.subprocess.run", side_effect=fake_run)
result = test_runner.run_pytest("src/tests", verbose=True,
with_coverage=False, capture_output=True)
assert len(result["failed_tests"]) == 1
error = result["failed_tests"][0]
assert error["id"] == "src/tests/test_b.py::test_needs_fixture"
assert error["message"] == "RuntimeError: boom"
assert error["location"] == "src/tests/test_b.py:3"
assert error["assert"] is None
def test_run_pytest_pins_no_color(self, mocker, test_runner):
"""Test the pytest command disables colors so parsing is stable.
Color-forcing environment variables (PY_COLORS, FORCE_COLOR)
would break the output parsers; the command pins --color=no.
"""
subprocess_mock = mocker.patch(
"fiona.run_tests.subprocess.run",
return_value=MagicMock(returncode=0, stdout="1 passed in 0.10s", stderr=""),
)
test_runner.run_pytest("src/tests", with_coverage=False, capture_output=True)
cmd = subprocess_mock.call_args[0][0]
assert "--color=no" in cmd
def test_run_pytest_warns_when_short_summary_missing(self, mocker, test_runner,
caplog):
"""Test a warning is logged when details come from junit only.
A missing pytest short summary (e.g. PYTEST_ADDOPTS=--no-summary
or a plugin suppressing it) means the regex parse finds nothing
and the junit report silently provides the details; the warning
makes the environment quirk visible.
"""
def fake_run(cmd, **kwargs):
self._write_junit_report(cmd[cmd.index("--junitxml") + 1], error=False)
return MagicMock(
returncode=1,
stdout=(
"src/tests/test_a.py::TestX::test_one FAILED [ 50%]\n"
"========================= 1 failed in 0.10s ========================\n"
),
stderr="",
)
mocker.patch("fiona.run_tests.subprocess.run", side_effect=fake_run)
with caplog.at_level(logging.WARNING, logger="fiona.run_tests"):
result = test_runner.run_pytest("src/tests", verbose=True,
with_coverage=False, capture_output=True)
assert result["failed_tests"][0]["assert"] == "assert 1 == 2"
assert "short summary missing" in caplog.text
def test_run_pytest_junit_no_duplicates_with_summary(self, mocker, test_runner):
"""Test junit entries do not duplicate short-summary entries."""
def fake_run(cmd, **kwargs):
self._write_junit_report(cmd[cmd.index("--junitxml") + 1], error=False)
return MagicMock(
returncode=1,
stdout=(
"src/tests/test_a.py::TestX::test_one FAILED [ 50%]\n"
"FAILED src/tests/test_a.py::TestX::test_one - AssertionError:"
" assert 1 == 2\n"
),
stderr="",
)
mocker.patch("fiona.run_tests.subprocess.run", side_effect=fake_run)
result = test_runner.run_pytest("src/tests", verbose=True,
with_coverage=False, capture_output=True)
assert len(result["failed_tests"]) == 1
assert result["failed_tests"][0]["id"] == "src/tests/test_a.py::TestX::test_one"
assert result["failed_tests"][0]["assert"] == "assert 1 == 2"
def test_run_pytest_junit_dedupes_across_path_forms(self, mocker, test_runner):
"""Test junit entries dedupe against summary ids with other paths.
pytest's junit report records `file` relative to its rootdir, so
the same test can appear under different path forms; the merge
must key on the file basename and test name, and prefer the
regex-parsed entry whose message matches pytest's own display.
"""
junit_xml = (
'<?xml version="1.0" encoding="utf-8"?>\n'
'<testsuites><testsuite name="pytest" errors="1" failures="0"'
' skipped="0" tests="1" time="0.1">\n'
'<testcase classname="test_integration_error" name="test_needs_fixture"'
' file="test_integration_error.py" line="3" time="0.001">'
'<error message="failed on setup with &quot;RuntimeError: boom&quot;"'
' type="test">traceback text</error></testcase>\n'
'</testsuite></testsuites>\n'
)
def fake_run(cmd, **kwargs):
pathlib.Path(cmd[cmd.index("--junitxml") + 1]).write_text(
junit_xml, encoding="utf-8"
)
return MagicMock(
returncode=1,
stdout=(
"../../tmp/test_integration_error.py::test_needs_fixture"
" ERROR [100%]\n"
"ERROR ../../tmp/test_integration_error.py::test_needs_fixture"
" - RuntimeError: boom\n"
),
stderr="",
)
mocker.patch("fiona.run_tests.subprocess.run", side_effect=fake_run)
result = test_runner.run_pytest("src/tests", verbose=True,
with_coverage=False, capture_output=True)
assert len(result["failed_tests"]) == 1
assert result["failed_tests"][0]["message"] == "RuntimeError: boom"
def test_run_pytest_junit_replaces_fallback_across_path_forms(self, mocker,
test_runner):
"""Test junit details replace generic fallback entries by basename.
In environments without a short summary the regex parse only
yields generic fallback entries; the junit report then provides
the real failure details, matched on the file basename and test
name even when the recorded paths differ.
"""
def fake_run(cmd, **kwargs):
self._write_junit_report(cmd[cmd.index("--junitxml") + 1], error=False)
return MagicMock(
returncode=1,
stdout=(
"../../tmp/test_a.py::TestX::test_one FAILED [100%]\n"
"========================= 1 failed in 0.10s ========================\n"
),
stderr="",
)
mocker.patch("fiona.run_tests.subprocess.run", side_effect=fake_run)
result = test_runner.run_pytest("src/tests", verbose=True,
with_coverage=False, capture_output=True)
assert len(result["failed_tests"]) == 1
failure = result["failed_tests"][0]
assert failure["message"] == "AssertionError: assert 1 == 2"
assert failure["location"] == "src/tests/test_a.py:10"
assert failure["assert"] == "assert 1 == 2"
def test_run_pytest_removes_junit_report(self, mocker, test_runner):
"""Test the temporary junit report is deleted after the run."""
written = {}
def fake_run(cmd, **kwargs):
junit_path = cmd[cmd.index("--junitxml") + 1]
self._write_junit_report(junit_path)
written["path"] = junit_path
return MagicMock(returncode=0, stdout="1 passed in 0.10s", stderr="")
mocker.patch("fiona.run_tests.subprocess.run", side_effect=fake_run)
test_runner.run_pytest("src/tests", with_coverage=False, capture_output=True)
assert not pathlib.Path(written["path"]).exists()
@patch('fiona.run_tests.subprocess.run')
def test_run_pytest_includes_failed_tests(self, mock_run, test_runner):
"""Test run_pytest returns the failed test details."""
mock_run.return_value = MagicMock(
returncode=1,
stdout=(
"src/tests/test_a.py::TestX::test_one FAILED [ 50%]\n"
"============================= 1 failed in 0.10s ==========================\n"
"FAILED src/tests/test_a.py::TestX::test_one - AssertionError: assert 1 == 2\n"
),
stderr=""
)
result = test_runner.run_pytest("src/tests", capture_output=True)
assert result["failed_tests"][0]["id"] == "src/tests/test_a.py::TestX::test_one"
assert "assert 1 == 2" in result["failed_tests"][0]["message"]
@patch('fiona.run_tests.subprocess.run')
def test_run_pytest_warns_when_failure_details_missing(self, mock_run, test_runner,
caplog):
"""Test a warning is logged when failures count but no details parse."""
mock_run.return_value = MagicMock(
returncode=1,
stdout=(
"============================= 1 failed in 0.10s ==========================\n"
),
stderr=""
)
with caplog.at_level(logging.WARNING, logger="fiona.run_tests"):
result = test_runner.run_pytest("src/tests", capture_output=True)
assert result["failed_count"] == 1
assert result["failed_tests"] == []
assert "no failure details" in caplog.text
@patch('fiona.run_tests.subprocess.run')
def test_run_pytest_includes_file_results(self, mock_run, test_runner):
"""Test run_pytest returns per-file results from verbose output."""
mock_run.return_value = MagicMock(
returncode=0,
stdout=(
"src/tests/test_a.py::TestX::test_one PASSED [ 10%]\n"
"src/tests/test_a.py::TestX::test_two PASSED [ 20%]\n"
"src/tests/test_b.py::TestY::test_three PASSED [ 30%]\n"
"============================= 3 passed in 0.10s ==========================\n"
),
stderr=""
)
result = test_runner.run_pytest("src/tests", verbose=True, capture_output=True)
assert result["file_results"]["test_a.py"]["passed"] == 2
assert result["file_results"]["test_b.py"]["passed"] == 1
@patch('fiona.run_tests.subprocess.run')
def test_run_pytest_includes_coverage_files(self, mock_run, test_runner):
"""Test run_pytest returns per-file coverage from the report."""
mock_run.return_value = MagicMock(
returncode=0,
stdout=(
"============================= 3 passed in 0.10s ==========================\n"
"Name Stmts Miss Cover\n"
"src/wiki/wiki.py 49 2 96%\n"
"TOTAL 798 27 97%\n"
),
stderr=""
)
result = test_runner.run_pytest("src/tests", capture_output=True)
assert result["coverage_files"]["src/wiki/wiki.py"] == 96.0
@patch('fiona.run_tests.subprocess.run')
def test_run_pytest_records_spawn_errors(self, mock_run, test_runner):
"""Test subprocess spawn failures are recorded, not raised."""
mock_run.side_effect = OSError("spawn failed")
result = test_runner.run_pytest("src/tests", capture_output=True)
assert result["passed"] is False
assert result["return_code"] == -1
assert result["exc"] is not None
assert test_runner.total_collected == 0
@patch('fiona.run_tests.subprocess.run')
def test_run_pytest_does_not_swallow_unexpected_errors(self, mock_run, test_runner):
"""Test unexpected exceptions propagate instead of being swallowed."""
mock_run.side_effect = RuntimeError("unexpected")
with pytest.raises(RuntimeError):
test_runner.run_pytest("src/tests", capture_output=True)
# @patch('fiona.run_tests.subprocess.run')
# def test_run_pytest_success(self, mock_run, test_runner):
# """Test running pytest successfully."""
# mock_run.return_value = MagicMock(
# returncode=0,
# stdout="=== 3 passed in 0.10s ===",
# stderr=""
# )
#
# with patch('builtins.print'):
# result = test_runner.run_pytest("src/tests", verbose=False, capture_output=False)
#
# assert result["passed"] is True
# assert result["return_code"] == 0
# assert result["passed_count"] == 3
# assert result["failed_count"] == 0
# assert test_runner.total_collected == 3
# assert test_runner.total_passed == 3
# assert test_runner.total_failed == 0
# @patch('fiona.run_tests.subprocess.run')
# def test_run_pytest_failure(self, mock_run, test_runner):
# """Test running pytest with failures."""
# mock_run.return_value = MagicMock(
# returncode=1,
# stdout="=== 2 passed, 1 failed in 0.10s ===",
# stderr=""
# )
#
# with patch('builtins.print'):
# result = test_runner.run_pytest("src/tests", verbose=False, capture_output=False)
#
# assert result["passed"] is False
# assert result["return_code"] == 1
# assert result["passed_count"] == 2
# assert result["failed_count"] == 1
# assert test_runner.total_collected == 3
# assert test_runner.total_passed == 2
# assert test_runner.total_failed == 1
# @patch('fiona.run_tests.subprocess.run')
# def test_run_pytest_exception(self, mock_run, test_runner):
# """Test handling of exceptions in pytest execution."""
# mock_run.side_effect = Exception("Test error")
#
# result = test_runner.run_pytest("src/tests", verbose=False, capture_output=False)
#
# assert result["passed"] is False
# assert result["return_code"] == -1
# assert result["stderr"] == "Test error"
# assert test_runner.total_collected == 0
# assert test_runner.total_passed == 0
# assert test_runner.total_failed == 0
# @patch('fiona.run_tests.subprocess.run')
# def test_run_pytest_verbose(self, mock_run, test_runner):
# """Test running pytest in verbose mode."""
# mock_run.return_value = MagicMock(
# returncode=0,
# stdout="=== 1 passed in 0.10s ===",
# stderr=""
# )
#
# with patch('builtins.print'):
# result = test_runner.run_pytest("src/tests", verbose=True, capture_output=False)
#
# mock_run.assert_called_once()
# call_args = mock_run.call_args[0][0]
# assert "-v" in call_args
# @patch('fiona.run_tests.subprocess.run')
# def test_run_pytest_capture_output(self, mock_run, test_runner):
# """Test running pytest with output capture."""
# mock_run.return_value = MagicMock(
# returncode=0,
# stdout="test output",
# stderr="test stderr"
# )
#
# result = test_runner.run_pytest("src/tests", verbose=False, capture_output=True)
#
# assert result["stdout"] == "test output"
# assert result["stderr"] == "test stderr"
def test_called_with_verbose_flag(self, mocker, test_runner):
"""Test that the pytest subprocess command includes the verbose flag."""
subprocess_mock = mocker.patch(
"fiona.run_tests.subprocess.run",
return_value=MagicMock(returncode=0, stdout="1 passed in 0.10s", stderr=""),
)
result = test_runner.run_pytest("src/tests", verbose=True, with_coverage=False)
cmd = subprocess_mock.call_args[0][0]
assert cmd[1:3] == ["-m", "pytest"]
assert "-v" in cmd
assert result["return_code"] == 0, f"run_pytest raised: {result}"
def test_prints_output_when_not_captured(self, mocker, test_runner, capsys):
"""Test that subprocess output is printed when capture is disabled."""
mocker.patch(
"fiona.run_tests.subprocess.run",
return_value=MagicMock(returncode=0, stdout="test output", stderr=""),
)
test_runner.run_pytest("src/tests", with_coverage=False, capture_output=False)
assert "test output" in capsys.readouterr().out
def test_suppresses_output_when_captured(self, mocker, test_runner, capsys):
"""Test that subprocess output is suppressed when capture is enabled."""
mocker.patch(
"fiona.run_tests.subprocess.run",
return_value=MagicMock(returncode=0, stdout="test output", stderr=""),
)
test_runner.run_pytest("src/tests", with_coverage=False, capture_output=True)
assert capsys.readouterr().out == ""
def test_records_coverage_percent(self, mocker, test_runner):
"""Test that a covered run records the coverage percentage from output."""
stdout = "TOTAL 729 98 87%\n3 passed in 0.10s"
mocker.patch(
"fiona.run_tests.subprocess.run",
return_value=MagicMock(returncode=0, stdout=stdout, stderr=""),
)
result = test_runner.run_pytest("src/tests", with_coverage=True)
assert result["coverage_percent"] == 87.0
assert test_runner.coverage_percent == 87.0
def test_coverage_percent_none_without_coverage(self, mocker, test_runner):
"""Test that coverage percentage stays None when coverage is disabled."""
mocker.patch(
"fiona.run_tests.subprocess.run",
return_value=MagicMock(returncode=0, stdout="1 passed in 0.10s", stderr=""),
)
result = test_runner.run_pytest("src/tests", with_coverage=False)
assert result["coverage_percent"] is None
def test_counts_parsed_from_output(self, mocker, test_runner):
"""Test that test counts are parsed from the subprocess output."""
stdout = "2 failed, 3 passed, 1 skipped in 0.10s"
mocker.patch(
"fiona.run_tests.subprocess.run",
return_value=MagicMock(returncode=1, stdout=stdout, stderr=""),
)
result = test_runner.run_pytest("src/tests", with_coverage=False)
assert result["passed_count"] == 3
assert result["failed_count"] == 2
assert result["skipped_count"] == 1
assert result["total_count"] == 6
assert result["passed"] is False
class TestRunAllTests:
"""Tests for TestRunner.run_all_tests."""
@patch.object(run_tests.TestRunner, 'run_pytest')
def test_run_all_tests_success(self, mock_run_pytest, test_runner):
"""Test running all tests successfully."""
mock_run_pytest.return_value = {
"passed": True,
"return_code": 0,
"passed_count": 3,
"failed_count": 0
}
with patch('builtins.print'):
all_passed = test_runner.run_all_tests(verbose=False, capture_output=False, with_coverage=True)
assert all_passed is True
mock_run_pytest.assert_called_once_with("tests", False, False, True)
@patch.object(run_tests.TestRunner, 'run_pytest')
def test_run_all_tests_failure(self, mock_run_pytest, test_runner):
"""Test running all tests with failures."""
mock_run_pytest.return_value = {
"passed": False,
"return_code": 1,
"passed_count": 0,
"failed_count": 1
}
with patch('builtins.print'):
all_passed = test_runner.run_all_tests(verbose=False, capture_output=False, with_coverage=True)
assert all_passed is False
class TestGetSummary:
"""Tests for TestRunner.get_summary."""
def test_get_summary_empty(self, test_runner):
"""Test getting summary with no tests run."""
summary = test_runner.get_summary()
assert summary["total_tests"] == 0
assert summary["passed_tests"] == 0
assert summary["failed_tests"] == 0
assert summary["test_details"] == {}
def test_get_summary_with_results(self, test_runner):
"""Test getting summary after running tests."""
test_runner.test_results["src/tests"] = {
"passed": True,
"passed_count": 3,
"failed_count": 0
}
test_runner.total_collected = 3
test_runner.total_passed = 3
test_runner.total_failed = 0
summary = test_runner.get_summary()
assert summary["total_tests"] == 3
assert summary["passed_tests"] == 3
assert summary["failed_tests"] == 0
assert "src/tests" in summary["test_details"]
class TestParseCoverageOutput:
"""Tests for TestRunner._parse_coverage_output."""
def test_parse_coverage_output_with_total(self, test_runner):
"""Test parsing coverage output with TOTAL line."""
output = "Name Stmts Miss Cover Missing\n-------------------------------------------\nTOTAL 100 20 80%"
result = test_runner._parse_coverage_output(output)
assert result == 80.0
def test_parse_coverage_output_with_total_coverage(self, test_runner):
"""Test parsing coverage output with 'Total coverage' line."""
output = "Total coverage: 95%"
result = test_runner._parse_coverage_output(output)
assert result == 95.0
def test_parse_coverage_output_empty(self, test_runner):
"""Test parsing empty coverage output."""
result = test_runner._parse_coverage_output("")
assert result == 0.0
def test_parse_coverage_output_no_match(self, test_runner):
"""Test parsing coverage output with no recognizable pattern."""
output = "Some random output without coverage percentage"
result = test_runner._parse_coverage_output(output)
assert result == 0.0
def test_parse_coverage_output_multiple_matches(self, test_runner):
"""Test parsing coverage output with multiple percentage values."""
output = "TOTAL 100 20 80%\nTotal coverage: 75%"
result = test_runner._parse_coverage_output(output)
# Should match the first pattern (TOTAL line)
assert result == 80.0
class TestRunTestsMain:
"""Tests for run_tests.run_tests.main() function."""
@patch('fiona.run_tests.TestRunner')
@patch('fiona.run_tests.argparse')
def test_main_all_tests_pass(self, mock_argparse, mock_runner_class):
"""Test run_tests.main() when all tests pass."""
mock_runner = MagicMock()
mock_runner.run_all_tests.return_value = True
mock_runner.get_summary.return_value = {
"total_tests": 5,
"passed_tests": 5,
"failed_tests": 0,
"coverage_percent": 90.0,
"test_details": {}
}
mock_runner_class.return_value = mock_runner
# Mock argparse to return our test args
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.verbose = False
mock_args.ci = False
mock_args.test = None
mock_args.skip_cov = False
mock_parser.parse_args.return_value = mock_args
mock_argparse.ArgumentParser.return_value = mock_parser
with patch('builtins.print'):
run_tests.main()
mock_runner.run_all_tests.assert_called_once_with(
False, False, True
)
@patch('fiona.run_tests.TestRunner')
@patch('fiona.run_tests.argparse')
def test_main_verbose_mode(self, mock_argparse, mock_runner_class):
"""Test run_tests.main() in verbose mode."""
mock_runner = MagicMock()
mock_runner.run_all_tests.return_value = True
mock_runner.get_summary.return_value = {
"total_tests": 5,
"passed_tests": 5,
"failed_tests": 0,
"coverage_percent": 90.0,
"test_details": {}
}
mock_runner_class.return_value = mock_runner
# Mock argparse to return our test args
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.verbose = True
mock_args.ci = False
mock_args.test = None
mock_args.skip_cov = False
mock_parser.parse_args.return_value = mock_args
mock_argparse.ArgumentParser.return_value = mock_parser
with patch('builtins.print'):
run_tests.main()
mock_runner.run_all_tests.assert_called_once_with(
True, False, True
)
@patch('fiona.run_tests.TestRunner')
@patch('fiona.run_tests.argparse')
def test_main_ci_mode(self, mock_argparse, mock_runner_class):
"""Test run_tests.main() in CI mode."""
mock_runner = MagicMock()
mock_runner.run_all_tests.return_value = True
mock_runner.get_summary.return_value = {
"total_tests": 5,
"passed_tests": 5,
"failed_tests": 0,
"coverage_percent": 90.0,
"test_details": {}
}
mock_runner_class.return_value = mock_runner
# Mock argparse to return our test args
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.verbose = False
mock_args.ci = True
mock_args.test = None
mock_args.skip_cov = False
mock_parser.parse_args.return_value = mock_args
mock_argparse.ArgumentParser.return_value = mock_parser
with patch('builtins.print'):
run_tests.main()
mock_runner.run_all_tests.assert_called_once_with(
False, True, True
)
@patch('fiona.run_tests.TestRunner')
@patch('fiona.run_tests.argparse')
def test_main_skip_coverage(self, mock_argparse, mock_runner_class):
"""Test run_tests.main() with --skip-cov flag."""
mock_runner = MagicMock()
mock_runner.run_all_tests.return_value = True
mock_runner.get_summary.return_value = {
"total_tests": 5,
"passed_tests": 5,
"failed_tests": 0,
"coverage_percent": None,
"test_details": {}
}
mock_runner_class.return_value = mock_runner
# Mock argparse to return our test args
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.verbose = False
mock_args.ci = False
mock_args.test = None
mock_args.skip_cov = True
mock_parser.parse_args.return_value = mock_args
mock_argparse.ArgumentParser.return_value = mock_parser
with patch('builtins.print'):
run_tests.main()
mock_runner.run_all_tests.assert_called_once_with(
False, False, False
)
@patch('fiona.run_tests.TestRunner')
@patch('fiona.run_tests.argparse')
@patch('fiona.run_tests.sys')
def test_main_specific_test(self, mock_sys, mock_argparse, mock_runner_class):
"""Test run_tests.main() with a specific test path."""
mock_runner = MagicMock()
mock_runner.run_pytest.return_value = {
"passed": True,
"return_code": 0,
"passed_count": 2,
"failed_count": 0
}
# Make get_summary return a proper dict with real integers
mock_runner.get_summary.return_value = {
"total_tests": 2,
"passed_tests": 2,
"failed_tests": 0,
"coverage_percent": None,
"test_details": {}
}
mock_runner_class.return_value = mock_runner
# Mock argparse to return our test args
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.verbose = False
mock_args.ci = False
mock_args.test = 'src/tests/test_librarian.py'
mock_args.skip_cov = False
mock_parser.parse_args.return_value = mock_args
mock_argparse.ArgumentParser.return_value = mock_parser
# Mock sys.exit to prevent actual exit
mock_sys.exit = MagicMock()
with patch('builtins.print'):
run_tests.main()
mock_runner.run_pytest.assert_called_once_with(
'src/tests/test_librarian.py', False, False, True
)
@patch('fiona.run_tests.TestRunner')
@patch('fiona.run_tests.argparse')
def test_main_tests_fail(self, mock_argparse, mock_runner_class):
"""Test run_tests.main() when tests fail."""
mock_runner = MagicMock()
mock_runner.run_all_tests.return_value = False
# Make get_summary return a proper dict with real integers
mock_runner.get_summary.return_value = {
"total_tests": 5,
"passed_tests": 3,
"failed_tests": 2, # Must be an int, not MagicMock
"coverage_percent": 90.0,
"test_details": {}
}
mock_runner_class.return_value = mock_runner
# Mock argparse to return our test args
mock_parser = MagicMock()
mock_args = MagicMock()
mock_args.verbose = False
mock_args.ci = False
mock_args.test = None
mock_args.skip_cov = False
mock_parser.parse_args.return_value = mock_args
mock_argparse.ArgumentParser.return_value = mock_parser
with patch('builtins.print'):
with pytest.raises(SystemExit) as exc_info:
run_tests.main()
assert exc_info.value.code == 1
class TestParseFailuresSection:
"""Tests for the FAILURES traceback section parser."""
def test_skips_blank_e_lines(self, test_runner):
"""Test blank E-lines do not add empty segments to the assert."""
output = (
"=================================== FAILURES ===================================\n"
"___________________ TestParseFrontmatter.test_parse_valid _____________________\n"
"\n"
" assert result[\"created\"] == \"2025-09-2\"\n"
"E AssertionError: assert '2025-09-20' == '2025-09-2'\n"
"E \n"
"E - 2025-09-2\n"
"E + 2025-09-20\n"
"\n"
"src/tests/test_md_frontmatter.py:217: AssertionError\n"
"=========================== short test summary info ============================\n"
"FAILED src/tests/test_md_frontmatter.py::TestParseFrontmatter::test_parse_valid - AssertionError\n"
)
failures = test_runner._parse_failed_tests_output(output)
assert failures[0]["location"] == "src/tests/test_md_frontmatter.py:217"
assert failures[0]["assert"] == "assert '2025-09-20' == '2025-09-2'"