Files
fiona-assistant/tests/test_sanity_runner.py

308 lines
9.3 KiB
Python

"""Tests for the shared SanityRunner engine (SPEC 7.14.1).
Covers the generalized class behind both the repository sanity runner
(run_sanity) and the wiki sanity command (wiki.sanity_manager):
concurrent check execution with per-check timings and the common
report formatting.
"""
from __future__ import annotations
import logging
import time
from fiona.sanity_runner import SanityRunner, format_duration
# =============================================================================
# Execution Tests
# =============================================================================
class TestRunAll:
"""Tests for the concurrent check runner."""
def test_runs_checks_concurrently(self):
def slow_check():
time.sleep(0.3)
return {"result": 0, "violation_count": 0}
runner = SanityRunner(checks=[
("a", slow_check),
("b", slow_check),
("c", slow_check),
])
start = time.perf_counter()
results = runner.run_all()
elapsed = time.perf_counter() - start
assert set(results) == {"a", "b", "c"}
# Sequential execution would take at least 0.9s
assert elapsed < 0.8
def test_results_keep_registry_order(self):
def quick_check():
return {"result": 0, "violation_count": 0}
def slow_check():
time.sleep(0.15)
return {"result": 0, "violation_count": 0}
runner = SanityRunner(checks=[("slow", slow_check), ("quick", quick_check)])
results = runner.run_all()
assert list(results) == ["slow", "quick"]
def test_durations_recorded(self):
runner = SanityRunner(checks=[("a", lambda: {"result": 0})])
results = runner.run_all()
assert "duration_seconds" in results["a"]
assert results["a"]["duration_seconds"] >= 0
def test_check_exception_propagates(self):
def broken_check():
raise RuntimeError("boom")
runner = SanityRunner(checks=[("a", broken_check)])
try:
runner.run_all()
except RuntimeError as error:
assert str(error) == "boom"
else:
raise AssertionError("expected RuntimeError to propagate")
class TestRunAllCallbacks:
"""Tests for the optional progress and heartbeat callbacks."""
def test_progress_receives_every_completed_check(self):
events = []
runner = SanityRunner(checks=[
("a", lambda: {"result": 0}),
("b", lambda: {"result": 1, "violation_count": 2}),
])
runner.run_all(progress=lambda name, check: events.append((name, check)))
assert sorted(name for name, _check in events) == ["a", "b"]
for _name, check in events:
assert "duration_seconds" in check
def test_heartbeat_receives_registry_ordered_remaining(self):
def slow_check():
time.sleep(0.7)
return {"result": 0}
def quick_check():
return {"result": 0}
runner = SanityRunner(checks=[("slow", slow_check), ("quick", quick_check)])
calls = []
runner.run_all(heartbeat=lambda remaining: calls.append(list(remaining)))
assert calls, "expected at least one heartbeat while the slow check ran"
assert all(call == ["slow"] for call in calls)
def test_no_heartbeat_once_all_checks_complete(self):
runner = SanityRunner(checks=[("a", lambda: {"result": 0})])
calls = []
runner.run_all(heartbeat=lambda remaining: calls.append(list(remaining)))
assert calls == []
def test_lifecycle_logged_at_info(self, caplog):
runner = SanityRunner(checks=[("a", lambda: {"result": 0})])
with caplog.at_level(logging.INFO, logger="fiona.sanity_runner.sanity_runner"):
runner.run_all()
messages = [record.message for record in caplog.records]
assert "check started: a" in messages
assert "check completed: a (00:00)" in messages
# =============================================================================
# Summary Tests
# =============================================================================
class TestFormatCheckSummary:
"""Tests for the one-line check summaries."""
def test_default_violation_count(self):
runner = SanityRunner()
assert runner.format_check_summary(
"ruff", {"result": 1, "violation_count": 21}
) == "FAIL - 21 violations"
assert runner.format_check_summary(
"markdownlint", {"result": 0, "violation_count": 0}
) == "PASS - 0 violations"
def test_duration_suffix(self):
runner = SanityRunner()
assert runner.format_check_summary("ruff", {
"result": 0,
"violation_count": 0,
"duration_seconds": 65,
}) == "PASS - 0 violations (01:05)"
def test_duration_omitted_when_missing(self):
runner = SanityRunner()
summary = runner.format_check_summary(
"ruff", {"result": 0, "violation_count": 0}
)
assert summary == "PASS - 0 violations"
def test_custom_summary_formatter(self):
runner = SanityRunner(summary_formatters={
"pytests": lambda check: f"{check.get('failed', 0)} tests failed",
})
assert runner.format_check_summary("pytests", {
"result": 0,
"failed": 0,
}) == "PASS - 0 tests failed"
class TestFormatDuration:
"""Tests for the MM:SS duration formatting."""
def test_zero(self):
assert format_duration(0) == "00:00"
def test_seconds(self):
assert format_duration(5) == "00:05"
def test_minutes(self):
assert format_duration(65.4) == "01:05"
def test_minutes_exceed_59(self):
assert format_duration(3661) == "61:01"
# =============================================================================
# Table, Banner, and Footer Tests
# =============================================================================
class TestFormatResultRows:
"""Tests for the aligned Results table rows."""
def test_aligned_rows(self):
runner = SanityRunner()
results = {
"markdownlint": {"result": 1, "violation_count": 2},
"ruff": {"result": 0, "violation_count": 0},
}
rows = runner.format_result_rows(results)
assert rows == [
" markdownlint: FAIL - 2 violations",
" ruff: PASS - 0 violations",
]
def test_extra_result_line_hook(self):
def coverage_line(name, check):
if name != "pytests":
return None
return ("coverage", {
"result": 0,
"coverage_percent": 95.0,
"required_coverage": 90.0,
"duration_seconds": None,
})
runner = SanityRunner(
summary_formatters={
"pytests": lambda check: f"{check.get('failed', 0)} tests failed",
"coverage": lambda check: (
f"{check['coverage_percent']}% "
f"(required {check['required_coverage']}%)"
),
},
extra_result_line=coverage_line,
)
results = {
"pytests": {
"result": 0,
"failed": 0,
"coverage_percent": 95.0,
"required_coverage": 90.0,
},
}
rows = runner.format_result_rows(results)
assert rows == [
" pytests: PASS - 0 tests failed",
" coverage: PASS - 95.0% (required 90.0%)",
]
class TestFormatPassBanner:
"""Tests for the SANITY PASSED banner."""
def test_banner_format(self):
runner = SanityRunner()
banner = runner.format_pass_banner(["b-check", "a-check"], 65)
assert banner.startswith("=" * 68)
assert "SANITY PASSED (01:05)" in banner
assert "b-check - a-check : all green" in banner
def test_sorted_names(self):
runner = SanityRunner()
banner = runner.format_pass_banner(["b-check", "a-check"], 5, sort_names=True)
assert "a-check - b-check : all green" in banner
class TestFormatFailureFooter:
"""Tests for the Sanity Failed footer."""
def test_single_check_unit(self):
runner = SanityRunner()
results = {
"ruff": {"result": 1, "violation_count": 21},
}
footer = runner.format_failure_footer(results)
assert footer == [
"",
"Sanity Failed - 1 check needs attention:",
" - ruff: 21 violations",
]
def test_multiple_checks_unit(self):
runner = SanityRunner()
results = {
"ruff": {"result": 1, "violation_count": 21},
"markdownlint": {"result": 1, "violation_count": 2},
}
footer = runner.format_failure_footer(results)
assert footer[1] == "Sanity Failed - 2 checks need attention:"
def test_failure_details_hook(self):
runner = SanityRunner(failure_details=lambda name, check: [
f" - detail for {name}" if name == "pytests" else ""
] if name == "pytests" else [])
results = {
"pytests": {"result": 1, "violation_count": 0},
}
footer = runner.format_failure_footer(results)
assert footer[-1] == " - detail for pytests"