Files

1326 lines
52 KiB
Python

"""Tests for the run_sanity.py sanity check runner."""
import io
import json
import logging
import pathlib
import re
from unittest.mock import patch
import pytest
import fiona.run_sanity as run_sanity
from fiona.customlinter.base import LintViolation
from fiona.customlinter.pycustomlint import PythonCustomLinter
from fiona.mdoctor.mdoctor import MDoctor
from fiona.python_ruff.python_ruff import PythonRuff
from fiona.run_tests import TestRunner
def _make_pytest_result(passed_count=1, failed_count=0, coverage_percent=95.0,
file_results=None, coverage_files=None, failed_tests=None):
"""Build a TestRunner.run_pytest result dict for testing."""
return {
"return_code": 0 if failed_count == 0 else 1,
"passed_count": passed_count,
"failed_count": failed_count,
"skipped_count": 0,
"total_count": passed_count + failed_count,
"passed": failed_count == 0,
"coverage_percent": coverage_percent,
"file_results": file_results if file_results is not None else {},
"coverage_files": coverage_files if coverage_files is not None else {},
"failed_tests": failed_tests if failed_tests is not None else [],
}
class TestLoadRequiredCoverage:
"""Tests for run_sanity.load_required_coverage."""
def test_load_required_coverage_default_settings(self):
"""Test the project settings file yields a valid coverage percentage."""
value = run_sanity.load_required_coverage()
assert isinstance(value, float)
assert 0 <= value <= 100
def test_load_required_coverage_custom_settings(self, tmp_path):
"""Test loading the required coverage from a custom settings file."""
settings_file = tmp_path / "settings.json"
settings_file.write_text('{"pytests": {"required-coverage": 75}}', encoding="utf-8")
assert run_sanity.load_required_coverage(settings_file) == 75
@pytest.fixture
def required_coverage_95(monkeypatch):
"""Pin the required coverage so tests do not track settings edits."""
monkeypatch.setattr(run_sanity, "load_required_coverage", lambda: 95.0)
@pytest.fixture
def mock_test_runner():
"""Patch TestRunner inside run_sanity to avoid running real tests."""
with patch("fiona.run_sanity.TestRunner") as runner_class:
yield runner_class
class TestRunPytests:
"""Tests for run_sanity.run_pytests."""
def test_pass_when_coverage_meets_required(self, mock_test_runner, required_coverage_95):
"""Test result is 0 when all tests pass and coverage meets the requirement."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result()
result = run_sanity.run_pytests()
assert result["result"] == 0
assert result["coverage_percent"] == 95.0
assert result["required_coverage"] == 95
def test_fail_when_coverage_below_required(self, mock_test_runner, required_coverage_95):
"""Test result is 1 when coverage is below the required percentage."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
coverage_percent=50.0
)
result = run_sanity.run_pytests()
assert result["result"] == 1
assert result["coverage_percent"] == 50.0
def test_fail_when_coverage_missing(self, mock_test_runner, required_coverage_95):
"""Test result is 1 when no coverage percentage is available."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
coverage_percent=None
)
result = run_sanity.run_pytests()
assert result["result"] == 1
assert result["coverage_percent"] is None
def test_fail_when_tests_fail(self, mock_test_runner, required_coverage_95):
"""Test result is 1 when any test fails, even with sufficient coverage."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
failed_count=1
)
result = run_sanity.run_pytests()
assert result["result"] == 1
assert result["failed"] == 1
def test_stdout_lists_files_and_coverage(self, mock_test_runner, required_coverage_95):
"""Test stdout lists the test files that ran and the coverage percent."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
coverage_percent=91.0,
file_results={
"test_a.py": {"passed": 2, "failed": 0, "skipped": 0, "total": 2},
"test_b.py": {"passed": 1, "failed": 1, "skipped": 0, "total": 2},
},
failed_tests=[
{
"id": "src/tests/test_b.py::TestX::test_one",
"message": "assert 1 == 2",
"location": "src/tests/test_b.py:5",
"assert": "assert 1 == 2",
},
],
)
result = run_sanity.run_pytests()
assert result["result"] == 1 # coverage 91.0% below required 95%
assert re.search(r"^ - test_a\.py\s+2/2 passed$",
result["stdout"], re.MULTILINE)
assert re.search(r"^ - test_b\.py\s+1/2 passed, 1 failed$",
result["stdout"], re.MULTILINE)
# the failed test nests under its file entry
assert (" - TestX : test_one\n"
" - assert 1 == 2") in result["stdout"]
assert "Coverage: 91.0% (required 95.0%)" in result["stdout"]
def test_stdout_when_no_file_results(self, mock_test_runner, required_coverage_95):
"""Test stdout still shows the coverage line without file details."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result()
result = run_sanity.run_pytests()
assert "Coverage: 95.0% (required 95.0%)" in result["stdout"]
assert "Tests: 1/1 passed" in result["stdout"]
def test_stdout_lists_coverage_per_file(self, mock_test_runner, required_coverage_95):
"""Test stdout lists the coverage percent for each source file."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
coverage_percent=97.0,
coverage_files={
"src/__init__.py": 100.0,
"src/mdoctor/mdoctor.py": 99.0,
"src/wiki/wiki.py": 96.0,
},
)
result = run_sanity.run_pytests()
assert "Coverage: 97.0% (required 95.0%)" in result["stdout"]
assert re.search(r"^ - __init__\.py\s+100\.0%$",
result["stdout"], re.MULTILINE)
assert re.search(r"^ - mdoctor/mdoctor\.py\s+99\.0%$",
result["stdout"], re.MULTILINE)
assert re.search(r"^ - wiki/wiki\.py\s+96\.0%$",
result["stdout"], re.MULTILINE)
def test_stdout_aligns_metrics_in_one_column(self, mock_test_runner, required_coverage_95):
"""Test test counts and coverage percents start at the same column."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
passed_count=3,
coverage_percent=97.0,
file_results={
"test_a.py": {"passed": 2, "failed": 0, "skipped": 0, "total": 2},
},
coverage_files={
"src/mdoctor/mdoctor.py": 99.0,
},
)
result = run_sanity.run_pytests()
detail_lines = [
line for line in result["stdout"].splitlines()
if line.startswith(" - ")
]
assert len(detail_lines) == 2
starts = {
line.index(line[len(" - "):].split(None, 1)[1])
for line in detail_lines
}
# every metric starts at the same column
assert len(starts) == 1
def test_run_pytests_logs_grouped_failures(self, mock_test_runner,
required_coverage_95, caplog):
"""Test the failure grouping is logged at debug level for debugging."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
failed_count=1,
coverage_percent=91.0,
file_results={
"test_a.py": {"passed": 1, "failed": 1, "skipped": 0, "total": 2},
},
failed_tests=[
{
"id": "src/tests/test_a.py::TestX::test_one",
"message": "assert 1 == 2",
"location": "src/tests/test_a.py:5",
"assert": "assert 1 == 2",
},
],
)
with caplog.at_level(logging.DEBUG, logger="fiona.run_sanity"):
run_sanity.run_pytests()
assert "src/tests/test_a.py::TestX::test_one" in caplog.text
def test_run_pytests_warns_when_file_results_show_unreported_failures(
self, mock_test_runner, required_coverage_95, caplog):
"""Test a warning is logged when files count failures without details.
This is the silent-failure case: the per-file counts say a test
failed but no failure details were parsed, so the report would
show "1 failed" without naming the test.
"""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
failed_count=1,
coverage_percent=91.0,
file_results={
"test_a.py": {"passed": 1, "failed": 1, "skipped": 0, "total": 2},
},
failed_tests=[],
)
with caplog.at_level(logging.WARNING, logger="fiona.run_sanity"):
run_sanity.run_pytests()
assert "no failure details" in caplog.text
def test_run_pytests_no_warning_when_details_outnumber_counts(
self, mock_test_runner, required_coverage_95, caplog):
"""Test no warning when more details than counted failures exist.
Duplicate detail entries for one test must not trip the
unreported-failures warning with a negative count.
"""
duplicate = {
"id": "src/tests/test_a.py::TestX::test_one",
"message": "assert 1 == 2",
"location": "src/tests/test_a.py:5",
"assert": "assert 1 == 2",
}
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
failed_count=1,
coverage_percent=91.0,
file_results={
"test_a.py": {"passed": 1, "failed": 1, "skipped": 0, "total": 2},
},
failed_tests=[duplicate, dict(duplicate)],
)
with caplog.at_level(logging.WARNING, logger="fiona.run_sanity"):
run_sanity.run_pytests()
assert "no failure details" not in caplog.text
def test_nested_explanation_falls_back_to_message_when_assert_missing(
self, mock_test_runner, required_coverage_95):
"""Test the nested explanation uses the message, not None.
Junit-sourced entries carry a location but no assert line for
non-assert errors; the nested explanation must fall back to
the message instead of printing "None".
"""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
failed_count=1,
coverage_percent=91.0,
file_results={
"test_a.py": {"passed": 1, "failed": 1, "skipped": 0, "total": 2},
},
failed_tests=[
{
"id": "src/tests/test_a.py::TestX::test_one",
"message": "TypeError: bad call",
"location": "src/tests/test_a.py:5",
"assert": None,
},
],
)
result = run_sanity.run_pytests()
assert (" - TestX : test_one\n"
" - TypeError: bad call") in result["stdout"]
assert "None" not in result["stdout"]
class TestConfigureLogging:
"""Tests for run_sanity.configure_logging."""
def test_log_level_env_sets_root_level(self, monkeypatch):
"""Test the LOG_LEVEL environment variable sets the logger levels.
The level is set on the root logger and on the module loggers
(run_sanity, run_tests) because pymarkdown resets the root
logger to WARNING during lint scans, which would otherwise
silence the debug output mid-run.
"""
root = logging.getLogger()
old_level = root.level
try:
monkeypatch.setenv("LOG_LEVEL", "DEBUG")
run_sanity.configure_logging()
assert root.level == logging.DEBUG
assert logging.getLogger("fiona.run_sanity").level == logging.DEBUG
assert logging.getLogger("fiona.run_tests").level == logging.DEBUG
monkeypatch.setenv("LOG_LEVEL", "INFO")
run_sanity.configure_logging()
assert root.level == logging.INFO
assert logging.getLogger("fiona.run_sanity").level == logging.INFO
assert logging.getLogger("fiona.run_tests").level == logging.INFO
finally:
root.setLevel(old_level)
for name in ("fiona.run_sanity", "fiona.run_tests"):
logging.getLogger(name).setLevel(logging.NOTSET)
def test_default_level_is_warning(self, monkeypatch):
"""Test the root logger defaults to WARNING without LOG_LEVEL."""
root = logging.getLogger()
old_level = root.level
try:
monkeypatch.delenv("LOG_LEVEL", raising=False)
run_sanity.configure_logging()
assert root.level == logging.WARNING
finally:
root.setLevel(old_level)
def _make_md_violation(scan_file=".", line=3, column=1,
rule_id="MD047", rule_name="single-trailing-newline",
description="Files should end with a single newline",
extra=""):
"""Build an MDoctor violation dict for testing."""
return {
"scan_file": scan_file,
"line": line,
"column": column,
"rule_id": rule_id,
"rule_name": rule_name,
"description": description,
"extra": extra,
}
@pytest.fixture
def mock_mdoctor():
"""Patch MDoctor inside run_sanity, keeping the real violation formatting."""
real_mdoctor = MDoctor()
with patch("fiona.run_sanity.MDoctor") as doctor_class:
instance = doctor_class.return_value
instance.compress_violations.side_effect = real_mdoctor.compress_violations
instance.format_violations.side_effect = real_mdoctor.format_violations
yield doctor_class
def _make_ruff_violation(filename, row=13, column=8,
code="F401", message="`argparse` imported but unused"):
"""Build a ruff JSON violation dict for testing."""
return {
"code": code,
"filename": filename,
"location": {"row": row, "column": column},
"end_location": {"row": row, "column": column + 10},
"message": message,
}
@pytest.fixture
def mock_python_ruff():
"""Patch PythonRuff inside run_sanity, keeping the real formatting."""
real_ruff = PythonRuff()
with patch("fiona.run_sanity.PythonRuff") as ruff_class:
instance = ruff_class.return_value
instance.compress_violations.side_effect = real_ruff.compress_violations
instance.format_violations.side_effect = real_ruff.format_violations
yield ruff_class
class TestRunRuff:
"""Tests for run_sanity.run_ruff."""
def test_stdout_lists_file_and_line(self, mock_python_ruff):
"""Test stdout names the violating file and the line:column position."""
filename = str(run_sanity.dev_settings_file().parent / "fiona.run_sanity.py")
mock_python_ruff.return_value.lint.return_value = [
_make_ruff_violation(filename, row=13, column=8, code="F401"),
]
result = run_sanity.run_ruff()
assert result["result"] == 1
assert result["violation_count"] == 1
assert "fiona.run_sanity.py" in result["stdout"]
assert "13:8 F401" in result["stdout"]
def test_no_violations_pass(self, mock_python_ruff):
"""Test result is 0 and stdout is present when no violations are found."""
mock_python_ruff.return_value.lint.return_value = []
result = run_sanity.run_ruff()
assert result["result"] == 0
assert result["violation_count"] == 0
assert "stdout" in result
def _make_pyc_violation(filename, line=205, column=4, code="PYC001",
message="Inline import: import argparse"):
"""Build a PythonCustomLinter LintViolation for testing."""
return LintViolation(
filepath=pathlib.Path(filename),
line=line,
column=column,
code=code,
message=message,
rule_id="inline-import",
)
@pytest.fixture
def mock_python_custom_linter():
"""Patch PythonCustomLinter inside run_sanity, keeping the real formatting."""
real_linter = PythonCustomLinter()
with patch("fiona.run_sanity.PythonCustomLinter") as linter_class:
instance = linter_class.return_value
instance.compress_violations.side_effect = real_linter.compress_violations
instance.format_compressed_violations.side_effect = (
real_linter.format_compressed_violations
)
yield linter_class
class TestRunPycustomlint:
"""Tests for run_sanity.run_pycustomlint."""
def test_stdout_uses_unified_report_format(self, mock_python_custom_linter):
"""Test stdout follows the shared linter report format (SPEC 6.1).
Like the markdownlint and frontmatter_lint reports, the
pycustomlint report carries the Violations header, TARGET,
DEFINITIONS, FILE, and SUMMARY sections.
"""
filename = str(
pathlib.Path("src/fiona/customlinter/pycustomlint.py").resolve()
)
mock_python_custom_linter.return_value.lint_directory.return_value = [
_make_pyc_violation(filename, line=205, column=4, code="PYC001",
message="PYC001 Inline import: import argparse"),
]
result = run_sanity.run_pycustomlint()
assert result["result"] == 1
assert result["violation_count"] == 1
stdout = result["stdout"]
assert "Python Custom Lint Violations" in stdout
assert "TARGET:" in stdout and "src" in stdout
assert "- DEFINITIONS:" in stdout
assert (
"- PYC001: Inline Import, Import statement found inside a function"
" or block (inline import). A module-level if/else before the"
" first function or class is allowed"
) in stdout
# File path is relative to target (src)
assert "- FILE: fiona/customlinter/pycustomlint.py" in stdout
assert " - PYC001 Inline import: import argparse" in stdout
assert "- SUMMARY:" in stdout
# file count first, matching the fmlint and markdownlint reports
assert stdout.index("- Total files with violations: 1") < \
stdout.index("- Total violations: 1")
def test_no_violations_pass(self, mock_python_custom_linter):
"""Test result is 0 and stdout is present when no violations are found."""
mock_python_custom_linter.return_value.lint_directory.return_value = []
result = run_sanity.run_pycustomlint()
assert result["result"] == 0
assert result["violation_count"] == 0
assert "stdout" in result
def test_scans_src_directory(self, mock_python_custom_linter):
"""Test the whole src tree is scanned."""
mock_python_custom_linter.return_value.lint_directory.return_value = []
run_sanity.run_pycustomlint()
mock_python_custom_linter.return_value.lint_directory.assert_called_once_with("src")
class TestRunMarkdownlint:
"""Tests for run_sanity.run_markdownlint."""
def test_stdout_lists_file_and_line(self, mock_mdoctor):
"""Test stdout names the violating file and the line:column position."""
mock_mdoctor.return_value.scan_file_programmatically.return_value = [
_make_md_violation(scan_file="llm-wiki/AGENTS.md", line=3),
]
result = run_sanity.run_markdownlint()
assert result["result"] == 1
assert result["violation_count"] == 1
assert "TARGET:" in result["stdout"]
assert "llm-wiki/AGENTS.md" in result["stdout"]
assert "3:1" in result["stdout"]
def test_no_violations_pass(self, mock_mdoctor):
"""Test result is 0 and stdout is present when no violations are found."""
mock_mdoctor.return_value.scan_file_programmatically.return_value = []
result = run_sanity.run_markdownlint()
assert result["result"] == 0
assert result["violation_count"] == 0
assert "stdout" in result
def test_scans_repo_root_recursively(self, mock_mdoctor):
"""Test the whole repo tree (src plus root docs) is scanned."""
mock_mdoctor.return_value.scan_file_programmatically.return_value = []
run_sanity.run_markdownlint()
mock_mdoctor.return_value.scan_file_programmatically.assert_called_once_with(".")
class TestMain:
"""Tests for run_sanity.main output."""
def test_prints_markdownlint_stdout_before_summary(self, capsys):
"""Test the AI report is printed before the result summary, once."""
results = {
"markdownlint": {
"result": 1,
"violation_count": 2,
"stdout": "AI-FORMATTED-VIOLATIONS",
},
"ruff": {"result": 0, "violation_count": 0},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert out.index("AI-FORMATTED-VIOLATIONS") < out.index("Results:")
assert "Sanity Failed" in out
assert out.count("AI-FORMATTED-VIOLATIONS") == 1
def test_prints_ruff_stdout_before_summary(self, capsys):
"""Test the ruff AI report is printed before the result summary, once."""
results = {
"ruff": {
"result": 1,
"violation_count": 1,
"stdout": "AI-RUFF-REPORT",
},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert out.index("AI-RUFF-REPORT") < out.index("Results:")
assert out.count("AI-RUFF-REPORT") == 1
def test_prints_pycustomlint_stdout_before_summary(self, capsys):
"""Test the pycustomlint AI report is printed before the result summary, once."""
results = {
"pycustomlint": {
"result": 1,
"violation_count": 3,
"stdout": "AI-PYCUSTOMLINT-REPORT",
},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert out.index("AI-PYCUSTOMLINT-REPORT") < out.index("Results:")
assert out.count("AI-PYCUSTOMLINT-REPORT") == 1
assert "Sanity Failed" in out
def test_prints_pytests_stdout_before_summary(self, capsys):
"""Test the pytests AI report is printed before the result summary."""
results = {
"pytests": {
"result": 1,
"passed": 193,
"failed": 0,
"total": 193,
"coverage_percent": 91.0,
"required_coverage": 95.0,
"stdout": "AI-PYTESTS-REPORT",
},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert out.index("AI-PYTESTS-REPORT") < out.index("Results:")
assert out.count("AI-PYTESTS-REPORT") == 1
assert "Sanity Failed" in out
def test_prints_pytests_report_when_clean(self, capsys):
"""Test the pytests report also prints on a clean run."""
results = {
"pytests": {
"result": 0,
"passed": 10,
"failed": 0,
"total": 10,
"coverage_percent": 95.0,
"required_coverage": 95.0,
"stdout": "AI-PYTESTS-REPORT",
},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert "AI-PYTESTS-REPORT" in out
assert "SANITY PASSED" in out
def test_no_markdownlint_report_when_clean(self, capsys):
"""Test nothing is printed for markdownlint when it has no violations."""
results = {
"markdownlint": {
"result": 0,
"violation_count": 0,
"stdout": "- SUMMARY:\n - Total violations: 0",
},
"ruff": {"result": 0, "violation_count": 0},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert "SUMMARY" not in out
assert "SANITY PASSED" in out
class TestFormatCheckSummary:
"""Tests for run_sanity.format_check_summary."""
def test_pytests_summary(self):
"""Test the passing pytests line reports zero failed tests."""
line = run_sanity.format_check_summary("pytests", {
"result": 0,
"passed": 189,
"failed": 0,
"total": 189,
"coverage_percent": 91.0,
"required_coverage": 90.0,
})
assert line == "PASS - 0 tests failed"
def test_coverage_summary(self):
"""Test the coverage line reports percent against the requirement."""
line = run_sanity.format_check_summary("coverage", {
"result": 0,
"coverage_percent": 97.0,
"required_coverage": 95.0,
})
assert line == "PASS - 97.0% (required 95.0%)"
line = run_sanity.format_check_summary("coverage", {
"result": 1,
"coverage_percent": 91.0,
"required_coverage": 95.0,
})
assert line == "FAIL - 91.0% (required 95.0%)"
def test_pytests_summary_failing(self):
"""Test the failing pytests line reports the failed test count."""
line = run_sanity.format_check_summary("pytests", {
"result": 1,
"passed": 5,
"failed": 2,
"total": 7,
"coverage_percent": 91.0,
"required_coverage": 90.0,
})
assert line == "FAIL - 2 tests failed"
def test_pytests_summary_one_test_failed(self):
"""Test the singular form for a single failing test."""
line = run_sanity.format_check_summary("pytests", {
"result": 1,
"passed": 231,
"failed": 1,
"total": 232,
"coverage_percent": 91.0,
"required_coverage": 90.0,
})
assert line == "FAIL - 1 test failed"
def test_pytests_summary_coverage_only_failure(self):
"""Test a coverage-only failure still reports zero failed tests."""
line = run_sanity.format_check_summary("pytests", {
"result": 1,
"passed": 232,
"failed": 0,
"total": 232,
"coverage_percent": 91.0,
"required_coverage": 95.0,
})
assert line == "FAIL - 0 tests failed"
def test_violation_check_summary(self):
"""Test lint checks report their violation count."""
assert run_sanity.format_check_summary(
"ruff", {"result": 1, "violation_count": 21}
) == "FAIL - 21 violations"
assert run_sanity.format_check_summary(
"markdownlint", {"result": 0, "violation_count": 0}
) == "PASS - 0 violations"
def test_format_duration(self):
"""Test durations render as zero-padded MM:SS."""
assert run_sanity.format_duration(0) == "00:00"
assert run_sanity.format_duration(5) == "00:05"
assert run_sanity.format_duration(65.4) == "01:05"
assert run_sanity.format_duration(3600) == "60:00"
assert run_sanity.format_duration(3661) == "61:01"
def test_summary_appends_duration_when_present(self):
"""Test summaries append (MM:SS) when duration_seconds is set."""
assert run_sanity.format_check_summary("ruff", {
"result": 0,
"violation_count": 0,
"duration_seconds": 65,
}) == "PASS - 0 violations (01:05)"
assert run_sanity.format_check_summary("pytests", {
"result": 0,
"passed": 10,
"failed": 0,
"total": 10,
"duration_seconds": 30,
}) == "PASS - 0 tests failed (00:30)"
def test_summary_omits_duration_when_missing(self):
"""Test summaries stay unchanged without duration_seconds."""
assert run_sanity.format_check_summary("ruff", {
"result": 0, "violation_count": 0
}) == "PASS - 0 violations"
def test_main_prints_aligned_summary_lines(self, capsys):
"""Test main() renders aligned per-check summary lines."""
results = {
"markdownlint": {
"result": 1,
"violation_count": 2,
"stdout": "AI-FORMATTED-VIOLATIONS",
},
"pytests": {
"result": 0,
"passed": 10,
"failed": 0,
"total": 10,
"coverage_percent": 95.0,
"required_coverage": 90.0,
},
"ruff": {"result": 0, "violation_count": 0},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert " markdownlint: FAIL - 2 violations" in out
assert " pytests: PASS - 0 tests failed" in out
assert " coverage: PASS - 95.0% (required 90.0%)" in out
assert " ruff: PASS - 0 violations" in out
assert "Sanity Failed" in out
class TestRunAllFunction:
"""Tests for run_sanity.run_all."""
def test_run_all_runs_all_checks(self, capsys):
"""Test run_all executes all six checks and returns their results."""
with patch("fiona.run_sanity.run_pytests", return_value={"result": 0}) as mock_p, \
patch("fiona.run_sanity.run_markdownlint",
return_value={"result": 0, "violation_count": 0}) as mock_m, \
patch("fiona.run_sanity.run_ruff",
return_value={"result": 0, "violation_count": 0}) as mock_r, \
patch("fiona.run_sanity.run_frontmatter_lint",
return_value={"result": 0, "violation_count": 0}) as mock_f, \
patch("fiona.run_sanity.run_pycustomlint",
return_value={"result": 0, "violation_count": 0}) as mock_c, \
patch("fiona.run_sanity.run_jsonlint",
return_value={"result": 0, "violation_count": 0}) as mock_j:
results = run_sanity.run_all()
assert set(results) == {
"pytests", "markdownlint", "ruff", "frontmatter_lint", "pycustomlint", "jsonlint"
}
assert all(r["result"] == 0 for r in results.values())
mock_p.assert_called_once()
mock_m.assert_called_once()
mock_r.assert_called_once()
mock_f.assert_called_once()
mock_c.assert_called_once()
mock_j.assert_called_once()
def test_run_all_records_durations(self, capsys):
"""Test run_all attaches a duration to every check result."""
with patch("fiona.run_sanity.run_pytests", return_value={"result": 0}), \
patch("fiona.run_sanity.run_markdownlint",
return_value={"result": 0, "violation_count": 0}), \
patch("fiona.run_sanity.run_ruff",
return_value={"result": 0, "violation_count": 0}), \
patch("fiona.run_sanity.run_frontmatter_lint",
return_value={"result": 0, "violation_count": 0}), \
patch("fiona.run_sanity.run_pycustomlint",
return_value={"result": 0, "violation_count": 0}):
results = run_sanity.run_all()
for check_result in results.values():
assert "duration_seconds" in check_result
assert check_result["duration_seconds"] >= 0
class TestVerdict:
"""Tests for the pass banner and the minimal fail verdict."""
def test_pass_prints_celebratory_banner(self, capsys):
"""Test a green run ends with a centered ASCII banner."""
results = {
"markdownlint": {"result": 0, "violation_count": 0},
"ruff": {"result": 0, "violation_count": 0},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert "SANITY PASSED" in out
assert "markdownlint - ruff : all green" in out
# banner bars above and below the title
assert out.count("=" * 20) >= 2
def test_pass_banner_includes_total_duration(self, capsys):
"""Test the green banner carries the total run time."""
results = {
"markdownlint": {"result": 0, "violation_count": 0},
"ruff": {"result": 0, "violation_count": 0},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert re.search(r"SANITY PASSED \(\d{2}:\d{2}\)", out)
def test_summary_lines_include_durations(self, capsys):
"""Test per-check lines print (MM:SS); coverage stays untimed."""
results = {
"pytests": {
"result": 0,
"passed": 10,
"failed": 0,
"total": 10,
"coverage_percent": 95.0,
"required_coverage": 90.0,
"duration_seconds": 42,
},
"ruff": {
"result": 0,
"violation_count": 0,
"duration_seconds": 7,
},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert " pytests: PASS - 0 tests failed (00:42)" in out
assert " ruff: PASS - 0 violations (00:07)" in out
assert " coverage: PASS - 95.0% (required 90.0%)" in out
def test_fail_verdict_lists_only_failures(self, capsys):
"""Test a failing run ends minimal, listing only failed checks."""
results = {
"markdownlint": {"result": 1, "violation_count": 12},
"pytests": {
"result": 0,
"passed": 10,
"failed": 0,
"total": 10,
"coverage_percent": 97.0,
"required_coverage": 95.0,
},
"ruff": {"result": 0, "violation_count": 0},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert "Sanity Failed - 1 check needs attention:" in out
assert " - markdownlint: 12 violations" in out
# passing checks are not repeated in the verdict
assert "pytests" not in out.split("Sanity Failed")[1]
def test_fail_verdict_plural_checks(self, capsys):
"""Test the fail verdict pluralizes when several checks fail."""
results = {
"markdownlint": {"result": 1, "violation_count": 12},
"ruff": {"result": 1, "violation_count": 3},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert "Sanity Failed - 2 checks need attention:" in out
assert " - markdownlint: 12 violations" in out
assert " - ruff: 3 violations" in out
def test_fail_verdict_shows_coverage_bullet(self, capsys):
"""Test a coverage shortfall is listed as its own verdict bullet."""
results = {
"pytests": {
"result": 1,
"passed": 220,
"failed": 0,
"total": 224,
"coverage_percent": 91.0,
"required_coverage": 95.0,
},
"ruff": {"result": 0, "violation_count": 0},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert " - pytests: 0 tests failed" in out
assert " - coverage: 91.0% (required 95.0%)" in out
def test_fail_verdict_reports_failed_test_count(self, capsys):
"""Test the verdict focuses on the number of failed tests."""
results = {
"pytests": {
"result": 1,
"passed": 231,
"failed": 1,
"total": 232,
"coverage_percent": 97.0,
"required_coverage": 96.0,
},
"markdownlint": {"result": 1, "violation_count": 2},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert " - pytests: 1 test failed" in out
assert " - markdownlint: 2 violations" in out
def test_stdout_lists_failed_tests(self, mock_test_runner, required_coverage_95):
"""Test failed tests nest under their file with name and assert."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
failed_count=1,
coverage_percent=91.0,
file_results={
"test_a.py": {"passed": 1, "failed": 1, "skipped": 0, "total": 2},
},
failed_tests=[
{
"id": "src/tests/test_a.py::TestX::test_one",
"message": "assert 1 == 2",
"location": "src/tests/test_a.py:5",
"assert": "assert 1 == 2",
},
],
)
result = run_sanity.run_pytests()
assert result["failed_tests"][0]["id"] == "src/tests/test_a.py::TestX::test_one"
assert (" - test_a.py 1/2 passed, 1 failed\n"
" - TestX : test_one\n"
" - assert 1 == 2") in result["stdout"]
# no consolidated Failures section: the nested block is the report
assert "Failures:" not in result["stdout"]
assert " - src/tests/test_a.py::TestX::test_one" not in result["stdout"]
def test_stdout_failed_test_without_location_falls_back(self, mock_test_runner,
required_coverage_95):
"""Test a failure without traceback details shows the raw message."""
mock_test_runner.return_value.run_pytest.return_value = _make_pytest_result(
failed_count=1,
coverage_percent=91.0,
file_results={
"test_a.py": {"passed": 1, "failed": 1, "skipped": 0, "total": 2},
},
failed_tests=[
{
"id": "src/tests/test_a.py::TestX::test_one",
"message": "TypeError: bad call",
"location": None,
"assert": None,
},
],
)
result = run_sanity.run_pytests()
assert " - TestX : test_one" in result["stdout"]
assert " - TypeError: bad call" in result["stdout"]
def test_fail_verdict_lists_failed_tests(self, capsys):
"""Test the verdict names each failed test without the assert.
The report body already shows the assert explanation nested
under the failing file; the verdict only names the tests.
"""
results = {
"pytests": {
"result": 1,
"passed": 242,
"failed": 1,
"total": 243,
"coverage_percent": 97.0,
"required_coverage": 96.0,
"failed_tests": [
{
"id": ("src/tests/test_temp_failure.py::"
"TestTempFailure::test_broken_assert"),
"message": "assert '1.0.0' == '2.0.0'",
"location": "src/tests/test_temp_failure.py:4",
"assert": "assert '1.0.0' == '2.0.0'",
},
],
},
}
with patch("fiona.run_sanity.run_all", return_value=results):
run_sanity.main([])
out = capsys.readouterr().out
assert " - pytests: 1 test failed" in out
assert " - TestTempFailure : test_broken_assert" in out
assert "assert '1.0.0' == '2.0.0'" not in out
assert "test_temp_failure.py:4" not in out
class TestFailedTestReportIntegration:
"""End-to-end checks that real failures reach the report."""
@staticmethod
def _patched_runner_for(test_file):
"""Patch run_sanity.TestRunner with a real runner for a test file.
The runner runs pytest in a real subprocess (no mocks, no coverage)
against the given file so the whole chain is exercised: real pytest
output, the failure parsing, and the report formatting in
run_pytests.
"""
class TargetedRunner(TestRunner):
"""Real TestRunner redirected to the given test file."""
def __init__(self, target):
super().__init__()
self.target = target
def run_pytest(self, test_path, verbose=False, capture_output=False,
with_coverage=True, ignore=None):
return super().run_pytest(
self.target, verbose, capture_output, with_coverage=False
)
return patch("fiona.run_sanity.TestRunner", lambda: TargetedRunner(test_file))
def test_real_failure_shows_function_and_assert(self, tmp_path):
"""Test a real pytest failure nests its name and assert under the file.
Runs the real TestRunner subprocess (no mocks) against a temporary
failing test so the whole chain is exercised: real pytest output,
the failure parsing, and the report formatting in run_pytests.
"""
test_file = tmp_path / "test_integration_failure.py"
test_file.write_text(
"class TestDemo:\n"
" def test_broken_assert(self):\n"
" assert 'Test Document' == 'Test Documen'\n",
encoding="utf-8",
)
with self._patched_runner_for(test_file):
result = run_sanity.run_pytests()
assert result["failed"] == 1
assert (" - test_integration_failure.py 0/1 passed, 1 failed\n"
" - TestDemo : test_broken_assert\n"
" - assert 'Test Document' == 'Test Documen'"
) in result["stdout"]
def test_real_error_shows_function_and_message(self, tmp_path):
"""Test a real pytest error (fixture failure) nests under the file.
A test that errors in a fixture is counted as failed in the file
listing; it must also show its name and error message instead of
vanishing from the report.
"""
test_file = tmp_path / "test_integration_error.py"
test_file.write_text(
"import pytest\n"
"\n"
"@pytest.fixture\n"
"def broken_fixture():\n"
" raise RuntimeError('boom')\n"
"\n"
"def test_needs_fixture(broken_fixture):\n"
" assert True\n",
encoding="utf-8",
)
with self._patched_runner_for(test_file):
result = run_sanity.run_pytests()
assert result["failed"] == 1
assert (" - test_integration_error.py 0/1 passed, 1 failed\n"
" - test_needs_fixture\n"
" - RuntimeError: boom"
) in result["stdout"]
def _fake_clock(times):
"""Build a clock returning each value of times once, then the last."""
values = iter(times)
state = {"now": times[0]}
def tick():
state["now"] = next(values, state["now"])
return state["now"]
return tick
class _FakeStream:
"""Minimal stream stub whose isatty() is configurable."""
def __init__(self, tty: bool):
self._tty = tty
def isatty(self) -> bool:
return self._tty
class TestFormatProgressBar:
"""Tests for the progress bar string rendering."""
def test_empty_bar_at_zero(self):
assert run_sanity.format_progress_bar(0.0) == "-" * 15
def test_partial_fill_rounds(self):
# 20% of 15 slots rounds to 3 filled slots
assert run_sanity.format_progress_bar(0.2) == "###------------"
def test_full_bar(self):
assert run_sanity.format_progress_bar(1.0) == "#" * 15
def test_fraction_capped_at_full(self):
assert run_sanity.format_progress_bar(2.5) == "#" * 15
def test_negative_fraction_capped_at_empty(self):
assert run_sanity.format_progress_bar(-0.5) == "-" * 15
class TestLoadProgressSettings:
"""Tests for the progress settings loader."""
def test_loads_interval_and_expected_seconds(self, tmp_path):
settings_file = tmp_path / "settings.json"
settings_file.write_text(
json.dumps({
"pytests": {"required-coverage": 96},
"progress": {"interval": 5, "expected-seconds": 45},
}),
encoding="utf-8",
)
loaded = run_sanity.load_progress_settings(settings_file)
assert loaded == {"interval_seconds": 5.0, "expected_seconds": 45.0}
def test_defaults_when_progress_section_missing(self, tmp_path):
settings_file = tmp_path / "settings.json"
settings_file.write_text(
json.dumps({"pytests": {"required-coverage": 96}}),
encoding="utf-8",
)
loaded = run_sanity.load_progress_settings(settings_file)
assert loaded == {"interval_seconds": 3.0, "expected_seconds": 30.0}
def test_partial_section_falls_back_per_key(self, tmp_path):
settings_file = tmp_path / "settings.json"
settings_file.write_text(
json.dumps({"progress": {"interval": 7}}),
encoding="utf-8",
)
loaded = run_sanity.load_progress_settings(settings_file)
assert loaded == {"interval_seconds": 7.0, "expected_seconds": 30.0}
class TestProgressDisplayNonInteractive:
"""Tests for the interval progress dots (non-interactive mode)."""
def test_no_print_before_interval_elapses(self):
stream = io.StringIO()
clock = _fake_clock([0.0, 0.5, 1.0])
display = run_sanity.ProgressDisplay(
["lint1", "lint2"], interactive=False, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
)
display.heartbeat(["lint1", "lint2"])
display.heartbeat(["lint1", "lint2"])
assert stream.getvalue() == ""
def test_prints_label_then_one_dot_per_interval(self):
stream = io.StringIO()
clock = _fake_clock([0.0, 3.2, 6.5, 7.0])
display = run_sanity.ProgressDisplay(
["lint1", "lint2"], interactive=False, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
)
display.heartbeat(["lint1", "lint2"])
assert stream.getvalue() == "Running Sanity ."
display.heartbeat(["lint1", "lint2"])
assert stream.getvalue() == "Running Sanity .."
# Below the interval again: no extra dot
display.heartbeat(["lint1", "lint2"])
assert stream.getvalue() == "Running Sanity .."
def test_custom_label_prefixes_the_dots(self):
stream = io.StringIO()
clock = _fake_clock([0.0, 3.2])
display = run_sanity.ProgressDisplay(
["lint1"], interactive=False, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
label="Running Wiki Sanity",
)
display.heartbeat(["lint1"])
assert stream.getvalue() == "Running Wiki Sanity ."
def test_finished_check_prints_nothing(self):
stream = io.StringIO()
clock = _fake_clock([0.0, 3.2])
display = run_sanity.ProgressDisplay(
["lint1", "lint2"], interactive=False, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
)
display.progress("lint1", {"result": 0, "duration_seconds": 7.12})
display.heartbeat(["lint2"])
# The dot comes from the interval tick, not from the finished
# check; no status lines or ANSI codes appear
assert stream.getvalue() == "Running Sanity ."
def test_finish_prints_newline_only_after_dots(self):
stream = io.StringIO()
clock = _fake_clock([0.0])
silent_display = run_sanity.ProgressDisplay(
["lint1"], interactive=False, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
)
silent_display.finish()
assert stream.getvalue() == ""
ticking_clock = _fake_clock([0.0, 3.2])
ticking_display = run_sanity.ProgressDisplay(
["lint1"], interactive=False, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=ticking_clock,
)
ticking_display.heartbeat(["lint1"])
ticking_display.finish()
assert stream.getvalue() == "Running Sanity .\n"
class TestProgressDisplayInteractive:
"""Tests for the in-place progress bars (interactive mode)."""
def test_renders_one_bar_per_check(self):
stream = io.StringIO()
clock = _fake_clock([0.0, 0.5])
display = run_sanity.ProgressDisplay(
["lint1", "lint2"], interactive=True, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
)
display.heartbeat(["lint1", "lint2"])
out = stream.getvalue()
assert out.count("\n") == 2
assert "lint1 [" in out
assert "lint2 [" in out
assert "%" in out
def test_redraw_moves_cursor_up_and_clears_lines(self):
stream = io.StringIO()
clock = _fake_clock([0.0, 0.5, 1.0])
display = run_sanity.ProgressDisplay(
["lint1", "lint2"], interactive=True, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
)
display.heartbeat(["lint1", "lint2"])
first_render = stream.getvalue()
assert "\x1b[2K" in first_render
display.heartbeat(["lint1", "lint2"])
assert stream.getvalue().startswith(first_render + "\x1b[2A")
def test_done_check_shows_full_bar_and_duration(self):
stream = io.StringIO()
clock = _fake_clock([0.0, 0.5, 1.0])
display = run_sanity.ProgressDisplay(
["lint1", "lint2"], interactive=True, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
)
display.heartbeat(["lint1", "lint2"])
display.progress("lint2", {"result": 0, "duration_seconds": 7.12})
out = stream.getvalue()
assert "lint2 " + "[" + "#" * 15 + "] 100% done (00:07)" in out
def test_finish_prints_separator_before_reports(self):
stream = io.StringIO()
clock = _fake_clock([0.0])
display = run_sanity.ProgressDisplay(
["lint1"], interactive=True, stream=stream,
interval_seconds=3.0, expected_seconds=30.0, clock=clock,
)
display.finish()
assert stream.getvalue() == "\n"
class TestNonInteractiveMode:
"""Tests for the --non-interactive argument and mode selection."""
def test_parse_args_defaults_to_interactive(self):
assert run_sanity.parse_args([]).non_interactive is False
def test_parse_args_accepts_non_interactive_flag(self):
assert run_sanity.parse_args(["--non-interactive"]).non_interactive is True
def test_should_use_interactive_requires_terminal(self):
assert run_sanity.should_use_interactive(False, _FakeStream(True)) is True
assert run_sanity.should_use_interactive(False, _FakeStream(False)) is False
assert run_sanity.should_use_interactive(True, _FakeStream(True)) is False
def test_main_wires_progress_callbacks(self, capsys):
"""Test main() passes the display callbacks to run_all."""
with patch("fiona.run_sanity.run_all", return_value={}) as mock_run_all:
run_sanity.main([])
assert mock_run_all.call_args.kwargs["progress"] is not None
assert mock_run_all.call_args.kwargs["heartbeat"] is not None