Files
fiona-assistant/tests/test_sanity_manager.py
T
2026-10-02 11:50:26 +02:00

398 lines
14 KiB
Python

"""Tests for the sanity_manager module (SPEC 7.1.3).
Covers the aggregated wiki sanity report (markdownlint, python-ruff,
mdlinter) with its modular check registry and the run_sanity-style
output structure (SPEC 6.1/7.14.1).
"""
from __future__ import annotations
import json
import re
import pytest
import fiona.run_sanity as run_sanity
from fiona.wiki import sanity_manager
from fiona.wiki.sanity_manager import (
format_sanity_report,
run_wiki_checks,
sanity_checks,
sanity_main,
wiki_sanity,
)
def _write_md(path, name: str, body: str, version: str = "1.0.0"):
"""Write a small md file with frontmatter at path."""
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
f"---\nname: {name}\nversion: {version}\n---\n\n{body}",
encoding="utf-8",
)
# =============================================================================
# Check Registry Tests
# =============================================================================
class TestSanityChecksRegistry:
"""Tests for the modular check registry."""
def test_registry_defines_the_wiki_checks(self):
names = {name for name, _runner in sanity_checks()}
assert names == {
"markdownlint",
"frontmatter_lint",
"python-ruff",
"mdlinter",
"jsonlint",
}
def test_registry_drives_results(self, pristine_check_results):
results = pristine_check_results
# Results keep the registry order regardless of completion order
assert list(results["checks"]) == [name for name, _runner in sanity_checks()]
# =============================================================================
# Aggregated Sanity Tests
# =============================================================================
class TestRunWikiChecks:
"""Tests for the aggregated check runner."""
def test_clean_wiki_passes(self, pristine_check_results):
results = pristine_check_results
assert results["passed"] is True
assert results["total_violations"] == 0
for check in results["checks"].values():
assert check["result"] == 0
assert check["violation_count"] == 0
assert check["stdout"]
assert "duration_seconds" in check
def test_dead_link_fails(self, minimal_wiki):
_write_md(
minimal_wiki / "wiki" / "concepts" / "broken.md",
"broken",
"# Broken\n\nSee [gone](../gone.md).\n",
)
results = run_wiki_checks(str(minimal_wiki))
assert results["passed"] is False
assert results["checks"]["mdlinter"]["result"] == 1
assert results["checks"]["mdlinter"]["violation_count"] == 1
assert "Markdown Lint Violations" in results["checks"]["mdlinter"]["stdout"]
def test_frontmatter_violation_fails(self, minimal_wiki):
# md file without frontmatter (project agent rule 10.1)
no_frontmatter = minimal_wiki / "wiki" / "concepts" / "no_frontmatter.md"
no_frontmatter.parent.mkdir(parents=True, exist_ok=True)
no_frontmatter.write_text(
"# Note\n\nNo frontmatter here.\n", encoding="utf-8"
)
results = run_wiki_checks(str(minimal_wiki))
assert results["passed"] is False
assert results["checks"]["frontmatter_lint"]["result"] == 1
assert results["checks"]["frontmatter_lint"]["violation_count"] > 0
assert (
"Frontmatter Lint Violations"
in results["checks"]["frontmatter_lint"]["stdout"]
)
def test_missing_directory_raises(self):
with pytest.raises(FileNotFoundError):
run_wiki_checks("/nonexistent/directory")
def test_non_wiki_directory_raises(self, tmp_path):
empty = tmp_path / "empty"
empty.mkdir()
with pytest.raises(NotADirectoryError):
run_wiki_checks(str(empty))
def test_path_rule_exemption_hides_markdownlint_rules(self, minimal_wiki):
"""SPEC 7.1.3.4: path_rule_exemptions filter wiki sanity markdownlint only."""
tasks = minimal_wiki / "tasks" / "tasks.md"
long_line = ("word " * 29) + "words"
_write_md(tasks, "tasks", f"# Tasks\n\n{long_line}\n")
without = run_wiki_checks(str(minimal_wiki))
assert without["checks"]["markdownlint"]["violation_count"] > 0
settings_path = minimal_wiki / "fiona-settings.json"
settings = json.loads(settings_path.read_text(encoding="utf-8"))
pymarkdown = settings.setdefault("markdown", {}).setdefault("pymarkdown", {})
pymarkdown["path_rule_exemptions"] = [
{"path": "tasks/**", "rules": ["MD013"]},
]
settings_path.write_text(json.dumps(settings), encoding="utf-8")
with_exempt = run_wiki_checks(str(minimal_wiki))
assert with_exempt["checks"]["markdownlint"]["violation_count"] == 0
def test_progress_callback_reports_every_check(self, minimal_wiki):
completed = []
run_wiki_checks(
str(minimal_wiki),
progress=lambda name, _result: completed.append(name),
)
assert set(completed) == {name for name, _check in sanity_checks()}
def test_heartbeat_callback_forwarded(self, minimal_wiki):
heartbeats = []
run_wiki_checks(
str(minimal_wiki), heartbeat=lambda remaining: heartbeats.append(list(remaining))
)
# The engine polls at least once while checks run; every poll
# carries the names of the still-running checks
for remaining in heartbeats:
assert set(remaining) <= {name for name, _check in sanity_checks()}
# =============================================================================
# Report Format Tests (run_sanity structure, SPEC 6.1/7.14.1)
# =============================================================================
class TestWikiSanityReport:
"""Integration tests for the human-readable report."""
def test_pristine_wiki_passes(self, minimal_wiki, capsys):
exit_code = wiki_sanity(str(minimal_wiki))
output = capsys.readouterr().out
assert exit_code == 0
assert "Results:" in output
assert re.search(r"markdownlint:\s+PASS - 0 violations", output)
assert re.search(r"python-ruff:\s+PASS - 0 violations", output)
assert re.search(r"mdlinter:\s+PASS - 0 violations", output)
assert "SANITY PASSED" in output
# Clean runs print no linter reports (SPEC 6.1)
assert "Violations\n" not in output
# Every summary line carries an (MM:SS) timing suffix (SPEC 7.14.1)
assert re.search(
r"mdlinter:\s+PASS - 0 violations \(\d{2}:\d{2}\)", output
)
def test_dead_link_fails(self, minimal_wiki, capsys):
_write_md(
minimal_wiki / "wiki" / "concepts" / "broken.md",
"broken",
"# Broken\n\nSee [gone](../gone.md).\n",
)
exit_code = wiki_sanity(str(minimal_wiki))
output = capsys.readouterr().out
assert exit_code == 1
# The failing check's report is printed before the Results table
assert output.index("Markdown Lint Violations") < output.index("Results:")
assert re.search(r"mdlinter:\s+FAIL - 1 violations", output)
assert "Sanity Failed - 1 check needs attention:" in output
assert "- mdlinter: 1 violations" in output
def test_markdown_violation_fails(self, minimal_wiki, capsys):
# MD013 ignores long lines without spaces, so use spaced words;
# the frontmatter keeps every other check clean so exactly one
# check fails
long_line = " ".join(["word"] * 40)
_write_md(
minimal_wiki / "wiki" / "concepts" / "long.md",
"long",
f"# Long\n\n{long_line}\n",
)
exit_code = wiki_sanity(str(minimal_wiki))
output = capsys.readouterr().out
assert exit_code == 1
assert re.search(r"markdownlint:\s+FAIL", output)
assert "Sanity Failed - 1 check needs attention:" in output
def test_python_violation_fails(self, minimal_wiki, capsys):
bad_py = minimal_wiki / "utils" / "scripts" / "bad.py"
bad_py.parent.mkdir(parents=True, exist_ok=True)
bad_py.write_text("import os\n", encoding="utf-8")
exit_code = wiki_sanity(str(minimal_wiki))
output = capsys.readouterr().out
assert exit_code == 1
assert re.search(r"python-ruff:\s+FAIL", output)
def test_format_sanity_report_structure(self, pristine_check_results):
report = format_sanity_report(pristine_check_results)
assert report.startswith("Wiki Sanity Report\n")
assert "Results:" in report
assert re.search(r"mdlinter:\s+PASS - 0 violations", report)
assert "SANITY PASSED" in report
# =============================================================================
# JSON Output Tests
# =============================================================================
class TestWikiSanityJson:
"""Tests for the JSON output."""
def test_json_reports_checks(self, minimal_wiki, capsys):
exit_code = wiki_sanity(str(minimal_wiki), json_output=True)
output = json.loads(capsys.readouterr().out)
assert exit_code == 0
assert output["passed"] is True
assert output["total_violations"] == 0
assert set(output["checks"]) == {
"markdownlint",
"frontmatter_lint",
"python-ruff",
"mdlinter",
"jsonlint",
}
for check in output["checks"].values():
assert check["result"] == 0
assert check["violation_count"] == 0
assert "duration_seconds" in check
def test_json_reports_dead_links(self, minimal_wiki, capsys):
_write_md(
minimal_wiki / "wiki" / "concepts" / "broken.md",
"broken",
"# Broken\n\nSee [gone](../gone.md).\n",
)
exit_code = wiki_sanity(str(minimal_wiki), json_output=True)
output = json.loads(capsys.readouterr().out)
assert exit_code == 1
assert output["passed"] is False
mdlinter = output["checks"]["mdlinter"]
assert mdlinter["result"] == 1
assert mdlinter["violation_count"] == 1
violation = mdlinter["violations"][0]
assert violation["code"] == "MDL001"
assert violation["line"] == 8
assert "../gone.md" in violation["message"]
assert violation["file"].endswith("broken.md")
# =============================================================================
# CLI Entry Tests
# =============================================================================
class TestSanityMain:
"""Tests for the CLI entry wrapper."""
def test_missing_directory_returns_one(self, capsys):
result = sanity_main(directory="/nonexistent/directory")
assert result == 1
assert "Error" in capsys.readouterr().err
def test_non_wiki_directory_returns_one(self, tmp_path, capsys):
empty = tmp_path / "empty"
empty.mkdir()
result = sanity_main(directory=str(empty))
assert result == 1
assert "Error" in capsys.readouterr().err
def test_forwards_non_interactive_to_wiki_sanity(self, minimal_wiki, monkeypatch):
received = {}
def fake_wiki_sanity(directory=".", json_output=False, non_interactive=False):
received["non_interactive"] = non_interactive
return 0
monkeypatch.setattr(sanity_manager, "wiki_sanity", fake_wiki_sanity)
result = sanity_main(directory=str(minimal_wiki), non_interactive=True)
assert result == 0
assert received["non_interactive"] is True
# =============================================================================
# Progress Display Tests (SPEC 7.14.2)
# =============================================================================
class TestWikiSanityProgress:
"""Tests for the in-run progress display wiring (SPEC 7.14.2)."""
@pytest.fixture
def fake_display_class(self, monkeypatch):
"""Replace run_sanity's ProgressDisplay with a recording stub."""
instances = []
class _StubDisplay:
def __init__(self, check_names, interactive, stream=None,
interval_seconds=3.0, expected_seconds=30.0,
clock=None, label="Running Sanity"):
self.check_names = list(check_names)
self.interactive = interactive
self.label = label
self.progress_calls = []
self.heartbeat_calls = []
self.finished = False
instances.append(self)
def progress(self, check_name, check_result):
self.progress_calls.append(check_name)
def heartbeat(self, remaining):
self.heartbeat_calls.append(list(remaining))
def finish(self):
self.finished = True
monkeypatch.setattr(run_sanity, "ProgressDisplay", _StubDisplay)
return instances
def test_display_covers_registered_checks(self, minimal_wiki, fake_display_class):
exit_code = wiki_sanity(str(minimal_wiki))
assert exit_code == 0
assert len(fake_display_class) == 1
display = fake_display_class[0]
assert display.check_names == [name for name, _check in sanity_checks()]
assert set(display.progress_calls) == {name for name, _check in sanity_checks()}
assert display.finished is True
def test_non_interactive_flag_drives_display_mode(
self, minimal_wiki, fake_display_class, monkeypatch
):
decisions = []
monkeypatch.setattr(
run_sanity, "should_use_interactive",
lambda flag, stream: decisions.append(flag) or flag,
)
wiki_sanity(str(minimal_wiki), non_interactive=True)
assert decisions == [True]
assert fake_display_class[0].interactive is True
def test_display_label_names_the_wiki_run(self, minimal_wiki, fake_display_class):
wiki_sanity(str(minimal_wiki))
assert fake_display_class[0].label == "Running Wiki Sanity"
def test_json_output_skips_display(self, minimal_wiki, fake_display_class, capsys):
exit_code = wiki_sanity(str(minimal_wiki), json_output=True)
assert exit_code == 0
assert fake_display_class == []
# The JSON stream stays pure: no progress noise around it
assert json.loads(capsys.readouterr().out)["passed"] is True