308 lines
9.3 KiB
Python
308 lines
9.3 KiB
Python
"""Tests for the shared SanityRunner engine (SPEC 7.14.1).
|
|
|
|
Covers the generalized class behind both the repository sanity runner
|
|
(run_sanity) and the wiki sanity command (wiki.sanity_manager):
|
|
concurrent check execution with per-check timings and the common
|
|
report formatting.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
|
|
from fiona.sanity_runner import SanityRunner, format_duration
|
|
|
|
# =============================================================================
|
|
# Execution Tests
|
|
# =============================================================================
|
|
|
|
class TestRunAll:
|
|
"""Tests for the concurrent check runner."""
|
|
|
|
def test_runs_checks_concurrently(self):
|
|
def slow_check():
|
|
time.sleep(0.3)
|
|
return {"result": 0, "violation_count": 0}
|
|
|
|
runner = SanityRunner(checks=[
|
|
("a", slow_check),
|
|
("b", slow_check),
|
|
("c", slow_check),
|
|
])
|
|
start = time.perf_counter()
|
|
results = runner.run_all()
|
|
elapsed = time.perf_counter() - start
|
|
|
|
assert set(results) == {"a", "b", "c"}
|
|
# Sequential execution would take at least 0.9s
|
|
assert elapsed < 0.8
|
|
|
|
def test_results_keep_registry_order(self):
|
|
def quick_check():
|
|
return {"result": 0, "violation_count": 0}
|
|
|
|
def slow_check():
|
|
time.sleep(0.15)
|
|
return {"result": 0, "violation_count": 0}
|
|
|
|
runner = SanityRunner(checks=[("slow", slow_check), ("quick", quick_check)])
|
|
|
|
results = runner.run_all()
|
|
|
|
assert list(results) == ["slow", "quick"]
|
|
|
|
def test_durations_recorded(self):
|
|
runner = SanityRunner(checks=[("a", lambda: {"result": 0})])
|
|
|
|
results = runner.run_all()
|
|
|
|
assert "duration_seconds" in results["a"]
|
|
assert results["a"]["duration_seconds"] >= 0
|
|
|
|
def test_check_exception_propagates(self):
|
|
def broken_check():
|
|
raise RuntimeError("boom")
|
|
|
|
runner = SanityRunner(checks=[("a", broken_check)])
|
|
|
|
try:
|
|
runner.run_all()
|
|
except RuntimeError as error:
|
|
assert str(error) == "boom"
|
|
else:
|
|
raise AssertionError("expected RuntimeError to propagate")
|
|
|
|
|
|
class TestRunAllCallbacks:
|
|
"""Tests for the optional progress and heartbeat callbacks."""
|
|
|
|
def test_progress_receives_every_completed_check(self):
|
|
events = []
|
|
runner = SanityRunner(checks=[
|
|
("a", lambda: {"result": 0}),
|
|
("b", lambda: {"result": 1, "violation_count": 2}),
|
|
])
|
|
|
|
runner.run_all(progress=lambda name, check: events.append((name, check)))
|
|
|
|
assert sorted(name for name, _check in events) == ["a", "b"]
|
|
for _name, check in events:
|
|
assert "duration_seconds" in check
|
|
|
|
def test_heartbeat_receives_registry_ordered_remaining(self):
|
|
def slow_check():
|
|
time.sleep(0.7)
|
|
return {"result": 0}
|
|
|
|
def quick_check():
|
|
return {"result": 0}
|
|
|
|
runner = SanityRunner(checks=[("slow", slow_check), ("quick", quick_check)])
|
|
calls = []
|
|
|
|
runner.run_all(heartbeat=lambda remaining: calls.append(list(remaining)))
|
|
|
|
assert calls, "expected at least one heartbeat while the slow check ran"
|
|
assert all(call == ["slow"] for call in calls)
|
|
|
|
def test_no_heartbeat_once_all_checks_complete(self):
|
|
runner = SanityRunner(checks=[("a", lambda: {"result": 0})])
|
|
calls = []
|
|
|
|
runner.run_all(heartbeat=lambda remaining: calls.append(list(remaining)))
|
|
|
|
assert calls == []
|
|
|
|
def test_lifecycle_logged_at_info(self, caplog):
|
|
runner = SanityRunner(checks=[("a", lambda: {"result": 0})])
|
|
|
|
with caplog.at_level(logging.INFO, logger="fiona.sanity_runner.sanity_runner"):
|
|
runner.run_all()
|
|
|
|
messages = [record.message for record in caplog.records]
|
|
assert "check started: a" in messages
|
|
assert "check completed: a (00:00)" in messages
|
|
|
|
|
|
# =============================================================================
|
|
# Summary Tests
|
|
# =============================================================================
|
|
|
|
class TestFormatCheckSummary:
|
|
"""Tests for the one-line check summaries."""
|
|
|
|
def test_default_violation_count(self):
|
|
runner = SanityRunner()
|
|
|
|
assert runner.format_check_summary(
|
|
"ruff", {"result": 1, "violation_count": 21}
|
|
) == "FAIL - 21 violations"
|
|
assert runner.format_check_summary(
|
|
"markdownlint", {"result": 0, "violation_count": 0}
|
|
) == "PASS - 0 violations"
|
|
|
|
def test_duration_suffix(self):
|
|
runner = SanityRunner()
|
|
|
|
assert runner.format_check_summary("ruff", {
|
|
"result": 0,
|
|
"violation_count": 0,
|
|
"duration_seconds": 65,
|
|
}) == "PASS - 0 violations (01:05)"
|
|
|
|
def test_duration_omitted_when_missing(self):
|
|
runner = SanityRunner()
|
|
|
|
summary = runner.format_check_summary(
|
|
"ruff", {"result": 0, "violation_count": 0}
|
|
)
|
|
|
|
assert summary == "PASS - 0 violations"
|
|
|
|
def test_custom_summary_formatter(self):
|
|
runner = SanityRunner(summary_formatters={
|
|
"pytests": lambda check: f"{check.get('failed', 0)} tests failed",
|
|
})
|
|
|
|
assert runner.format_check_summary("pytests", {
|
|
"result": 0,
|
|
"failed": 0,
|
|
}) == "PASS - 0 tests failed"
|
|
|
|
|
|
class TestFormatDuration:
|
|
"""Tests for the MM:SS duration formatting."""
|
|
|
|
def test_zero(self):
|
|
assert format_duration(0) == "00:00"
|
|
|
|
def test_seconds(self):
|
|
assert format_duration(5) == "00:05"
|
|
|
|
def test_minutes(self):
|
|
assert format_duration(65.4) == "01:05"
|
|
|
|
def test_minutes_exceed_59(self):
|
|
assert format_duration(3661) == "61:01"
|
|
|
|
|
|
# =============================================================================
|
|
# Table, Banner, and Footer Tests
|
|
# =============================================================================
|
|
|
|
class TestFormatResultRows:
|
|
"""Tests for the aligned Results table rows."""
|
|
|
|
def test_aligned_rows(self):
|
|
runner = SanityRunner()
|
|
results = {
|
|
"markdownlint": {"result": 1, "violation_count": 2},
|
|
"ruff": {"result": 0, "violation_count": 0},
|
|
}
|
|
|
|
rows = runner.format_result_rows(results)
|
|
|
|
assert rows == [
|
|
" markdownlint: FAIL - 2 violations",
|
|
" ruff: PASS - 0 violations",
|
|
]
|
|
|
|
def test_extra_result_line_hook(self):
|
|
def coverage_line(name, check):
|
|
if name != "pytests":
|
|
return None
|
|
return ("coverage", {
|
|
"result": 0,
|
|
"coverage_percent": 95.0,
|
|
"required_coverage": 90.0,
|
|
"duration_seconds": None,
|
|
})
|
|
|
|
runner = SanityRunner(
|
|
summary_formatters={
|
|
"pytests": lambda check: f"{check.get('failed', 0)} tests failed",
|
|
"coverage": lambda check: (
|
|
f"{check['coverage_percent']}% "
|
|
f"(required {check['required_coverage']}%)"
|
|
),
|
|
},
|
|
extra_result_line=coverage_line,
|
|
)
|
|
results = {
|
|
"pytests": {
|
|
"result": 0,
|
|
"failed": 0,
|
|
"coverage_percent": 95.0,
|
|
"required_coverage": 90.0,
|
|
},
|
|
}
|
|
|
|
rows = runner.format_result_rows(results)
|
|
|
|
assert rows == [
|
|
" pytests: PASS - 0 tests failed",
|
|
" coverage: PASS - 95.0% (required 90.0%)",
|
|
]
|
|
|
|
|
|
class TestFormatPassBanner:
|
|
"""Tests for the SANITY PASSED banner."""
|
|
|
|
def test_banner_format(self):
|
|
runner = SanityRunner()
|
|
|
|
banner = runner.format_pass_banner(["b-check", "a-check"], 65)
|
|
|
|
assert banner.startswith("=" * 68)
|
|
assert "SANITY PASSED (01:05)" in banner
|
|
assert "b-check - a-check : all green" in banner
|
|
|
|
def test_sorted_names(self):
|
|
runner = SanityRunner()
|
|
|
|
banner = runner.format_pass_banner(["b-check", "a-check"], 5, sort_names=True)
|
|
|
|
assert "a-check - b-check : all green" in banner
|
|
|
|
|
|
class TestFormatFailureFooter:
|
|
"""Tests for the Sanity Failed footer."""
|
|
|
|
def test_single_check_unit(self):
|
|
runner = SanityRunner()
|
|
results = {
|
|
"ruff": {"result": 1, "violation_count": 21},
|
|
}
|
|
|
|
footer = runner.format_failure_footer(results)
|
|
|
|
assert footer == [
|
|
"",
|
|
"Sanity Failed - 1 check needs attention:",
|
|
" - ruff: 21 violations",
|
|
]
|
|
|
|
def test_multiple_checks_unit(self):
|
|
runner = SanityRunner()
|
|
results = {
|
|
"ruff": {"result": 1, "violation_count": 21},
|
|
"markdownlint": {"result": 1, "violation_count": 2},
|
|
}
|
|
|
|
footer = runner.format_failure_footer(results)
|
|
|
|
assert footer[1] == "Sanity Failed - 2 checks need attention:"
|
|
|
|
def test_failure_details_hook(self):
|
|
runner = SanityRunner(failure_details=lambda name, check: [
|
|
f" - detail for {name}" if name == "pytests" else ""
|
|
] if name == "pytests" else [])
|
|
results = {
|
|
"pytests": {"result": 1, "violation_count": 0},
|
|
}
|
|
|
|
footer = runner.format_failure_footer(results)
|
|
|
|
assert footer[-1] == " - detail for pytests"
|