398 lines
14 KiB
Python
398 lines
14 KiB
Python
"""Tests for the sanity_manager module (SPEC 7.1.3).
|
|
|
|
Covers the aggregated wiki sanity report (markdownlint, python-ruff,
|
|
mdlinter) with its modular check registry and the run_sanity-style
|
|
output structure (SPEC 6.1/7.14.1).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
|
|
import pytest
|
|
|
|
import fiona.run_sanity as run_sanity
|
|
from fiona.wiki import sanity_manager
|
|
from fiona.wiki.sanity_manager import (
|
|
format_sanity_report,
|
|
run_wiki_checks,
|
|
sanity_checks,
|
|
sanity_main,
|
|
wiki_sanity,
|
|
)
|
|
|
|
|
|
def _write_md(path, name: str, body: str, version: str = "1.0.0"):
|
|
"""Write a small md file with frontmatter at path."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(
|
|
f"---\nname: {name}\nversion: {version}\n---\n\n{body}",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Check Registry Tests
|
|
# =============================================================================
|
|
|
|
class TestSanityChecksRegistry:
|
|
"""Tests for the modular check registry."""
|
|
|
|
def test_registry_defines_the_wiki_checks(self):
|
|
names = {name for name, _runner in sanity_checks()}
|
|
|
|
assert names == {
|
|
"markdownlint",
|
|
"frontmatter_lint",
|
|
"python-ruff",
|
|
"mdlinter",
|
|
"jsonlint",
|
|
}
|
|
|
|
def test_registry_drives_results(self, pristine_check_results):
|
|
results = pristine_check_results
|
|
|
|
# Results keep the registry order regardless of completion order
|
|
assert list(results["checks"]) == [name for name, _runner in sanity_checks()]
|
|
|
|
|
|
# =============================================================================
|
|
# Aggregated Sanity Tests
|
|
# =============================================================================
|
|
|
|
class TestRunWikiChecks:
|
|
"""Tests for the aggregated check runner."""
|
|
|
|
def test_clean_wiki_passes(self, pristine_check_results):
|
|
results = pristine_check_results
|
|
|
|
assert results["passed"] is True
|
|
assert results["total_violations"] == 0
|
|
for check in results["checks"].values():
|
|
assert check["result"] == 0
|
|
assert check["violation_count"] == 0
|
|
assert check["stdout"]
|
|
assert "duration_seconds" in check
|
|
|
|
def test_dead_link_fails(self, minimal_wiki):
|
|
_write_md(
|
|
minimal_wiki / "wiki" / "concepts" / "broken.md",
|
|
"broken",
|
|
"# Broken\n\nSee [gone](../gone.md).\n",
|
|
)
|
|
|
|
results = run_wiki_checks(str(minimal_wiki))
|
|
|
|
assert results["passed"] is False
|
|
assert results["checks"]["mdlinter"]["result"] == 1
|
|
assert results["checks"]["mdlinter"]["violation_count"] == 1
|
|
assert "Markdown Lint Violations" in results["checks"]["mdlinter"]["stdout"]
|
|
|
|
def test_frontmatter_violation_fails(self, minimal_wiki):
|
|
# md file without frontmatter (project agent rule 10.1)
|
|
no_frontmatter = minimal_wiki / "wiki" / "concepts" / "no_frontmatter.md"
|
|
no_frontmatter.parent.mkdir(parents=True, exist_ok=True)
|
|
no_frontmatter.write_text(
|
|
"# Note\n\nNo frontmatter here.\n", encoding="utf-8"
|
|
)
|
|
|
|
results = run_wiki_checks(str(minimal_wiki))
|
|
|
|
assert results["passed"] is False
|
|
assert results["checks"]["frontmatter_lint"]["result"] == 1
|
|
assert results["checks"]["frontmatter_lint"]["violation_count"] > 0
|
|
assert (
|
|
"Frontmatter Lint Violations"
|
|
in results["checks"]["frontmatter_lint"]["stdout"]
|
|
)
|
|
|
|
def test_missing_directory_raises(self):
|
|
with pytest.raises(FileNotFoundError):
|
|
run_wiki_checks("/nonexistent/directory")
|
|
|
|
def test_non_wiki_directory_raises(self, tmp_path):
|
|
empty = tmp_path / "empty"
|
|
empty.mkdir()
|
|
|
|
with pytest.raises(NotADirectoryError):
|
|
run_wiki_checks(str(empty))
|
|
|
|
def test_path_rule_exemption_hides_markdownlint_rules(self, minimal_wiki):
|
|
"""SPEC 7.1.3.4: path_rule_exemptions filter wiki sanity markdownlint only."""
|
|
tasks = minimal_wiki / "tasks" / "tasks.md"
|
|
long_line = ("word " * 29) + "words"
|
|
_write_md(tasks, "tasks", f"# Tasks\n\n{long_line}\n")
|
|
|
|
without = run_wiki_checks(str(minimal_wiki))
|
|
assert without["checks"]["markdownlint"]["violation_count"] > 0
|
|
|
|
settings_path = minimal_wiki / "fiona-settings.json"
|
|
settings = json.loads(settings_path.read_text(encoding="utf-8"))
|
|
pymarkdown = settings.setdefault("markdown", {}).setdefault("pymarkdown", {})
|
|
pymarkdown["path_rule_exemptions"] = [
|
|
{"path": "tasks/**", "rules": ["MD013"]},
|
|
]
|
|
settings_path.write_text(json.dumps(settings), encoding="utf-8")
|
|
|
|
with_exempt = run_wiki_checks(str(minimal_wiki))
|
|
assert with_exempt["checks"]["markdownlint"]["violation_count"] == 0
|
|
|
|
def test_progress_callback_reports_every_check(self, minimal_wiki):
|
|
completed = []
|
|
run_wiki_checks(
|
|
str(minimal_wiki),
|
|
progress=lambda name, _result: completed.append(name),
|
|
)
|
|
|
|
assert set(completed) == {name for name, _check in sanity_checks()}
|
|
|
|
def test_heartbeat_callback_forwarded(self, minimal_wiki):
|
|
heartbeats = []
|
|
run_wiki_checks(
|
|
str(minimal_wiki), heartbeat=lambda remaining: heartbeats.append(list(remaining))
|
|
)
|
|
|
|
# The engine polls at least once while checks run; every poll
|
|
# carries the names of the still-running checks
|
|
for remaining in heartbeats:
|
|
assert set(remaining) <= {name for name, _check in sanity_checks()}
|
|
|
|
|
|
# =============================================================================
|
|
# Report Format Tests (run_sanity structure, SPEC 6.1/7.14.1)
|
|
# =============================================================================
|
|
|
|
class TestWikiSanityReport:
|
|
"""Integration tests for the human-readable report."""
|
|
|
|
def test_pristine_wiki_passes(self, minimal_wiki, capsys):
|
|
exit_code = wiki_sanity(str(minimal_wiki))
|
|
output = capsys.readouterr().out
|
|
|
|
assert exit_code == 0
|
|
assert "Results:" in output
|
|
assert re.search(r"markdownlint:\s+PASS - 0 violations", output)
|
|
assert re.search(r"python-ruff:\s+PASS - 0 violations", output)
|
|
assert re.search(r"mdlinter:\s+PASS - 0 violations", output)
|
|
assert "SANITY PASSED" in output
|
|
# Clean runs print no linter reports (SPEC 6.1)
|
|
assert "Violations\n" not in output
|
|
# Every summary line carries an (MM:SS) timing suffix (SPEC 7.14.1)
|
|
assert re.search(
|
|
r"mdlinter:\s+PASS - 0 violations \(\d{2}:\d{2}\)", output
|
|
)
|
|
|
|
def test_dead_link_fails(self, minimal_wiki, capsys):
|
|
_write_md(
|
|
minimal_wiki / "wiki" / "concepts" / "broken.md",
|
|
"broken",
|
|
"# Broken\n\nSee [gone](../gone.md).\n",
|
|
)
|
|
|
|
exit_code = wiki_sanity(str(minimal_wiki))
|
|
output = capsys.readouterr().out
|
|
|
|
assert exit_code == 1
|
|
# The failing check's report is printed before the Results table
|
|
assert output.index("Markdown Lint Violations") < output.index("Results:")
|
|
assert re.search(r"mdlinter:\s+FAIL - 1 violations", output)
|
|
assert "Sanity Failed - 1 check needs attention:" in output
|
|
assert "- mdlinter: 1 violations" in output
|
|
|
|
def test_markdown_violation_fails(self, minimal_wiki, capsys):
|
|
# MD013 ignores long lines without spaces, so use spaced words;
|
|
# the frontmatter keeps every other check clean so exactly one
|
|
# check fails
|
|
long_line = " ".join(["word"] * 40)
|
|
_write_md(
|
|
minimal_wiki / "wiki" / "concepts" / "long.md",
|
|
"long",
|
|
f"# Long\n\n{long_line}\n",
|
|
)
|
|
|
|
exit_code = wiki_sanity(str(minimal_wiki))
|
|
output = capsys.readouterr().out
|
|
|
|
assert exit_code == 1
|
|
assert re.search(r"markdownlint:\s+FAIL", output)
|
|
assert "Sanity Failed - 1 check needs attention:" in output
|
|
|
|
def test_python_violation_fails(self, minimal_wiki, capsys):
|
|
bad_py = minimal_wiki / "utils" / "scripts" / "bad.py"
|
|
bad_py.parent.mkdir(parents=True, exist_ok=True)
|
|
bad_py.write_text("import os\n", encoding="utf-8")
|
|
|
|
exit_code = wiki_sanity(str(minimal_wiki))
|
|
output = capsys.readouterr().out
|
|
|
|
assert exit_code == 1
|
|
assert re.search(r"python-ruff:\s+FAIL", output)
|
|
|
|
def test_format_sanity_report_structure(self, pristine_check_results):
|
|
report = format_sanity_report(pristine_check_results)
|
|
|
|
assert report.startswith("Wiki Sanity Report\n")
|
|
assert "Results:" in report
|
|
assert re.search(r"mdlinter:\s+PASS - 0 violations", report)
|
|
assert "SANITY PASSED" in report
|
|
|
|
|
|
# =============================================================================
|
|
# JSON Output Tests
|
|
# =============================================================================
|
|
|
|
class TestWikiSanityJson:
|
|
"""Tests for the JSON output."""
|
|
|
|
def test_json_reports_checks(self, minimal_wiki, capsys):
|
|
exit_code = wiki_sanity(str(minimal_wiki), json_output=True)
|
|
output = json.loads(capsys.readouterr().out)
|
|
|
|
assert exit_code == 0
|
|
assert output["passed"] is True
|
|
assert output["total_violations"] == 0
|
|
assert set(output["checks"]) == {
|
|
"markdownlint",
|
|
"frontmatter_lint",
|
|
"python-ruff",
|
|
"mdlinter",
|
|
"jsonlint",
|
|
}
|
|
for check in output["checks"].values():
|
|
assert check["result"] == 0
|
|
assert check["violation_count"] == 0
|
|
assert "duration_seconds" in check
|
|
|
|
def test_json_reports_dead_links(self, minimal_wiki, capsys):
|
|
_write_md(
|
|
minimal_wiki / "wiki" / "concepts" / "broken.md",
|
|
"broken",
|
|
"# Broken\n\nSee [gone](../gone.md).\n",
|
|
)
|
|
|
|
exit_code = wiki_sanity(str(minimal_wiki), json_output=True)
|
|
output = json.loads(capsys.readouterr().out)
|
|
|
|
assert exit_code == 1
|
|
assert output["passed"] is False
|
|
mdlinter = output["checks"]["mdlinter"]
|
|
assert mdlinter["result"] == 1
|
|
assert mdlinter["violation_count"] == 1
|
|
violation = mdlinter["violations"][0]
|
|
assert violation["code"] == "MDL001"
|
|
assert violation["line"] == 8
|
|
assert "../gone.md" in violation["message"]
|
|
assert violation["file"].endswith("broken.md")
|
|
|
|
|
|
# =============================================================================
|
|
# CLI Entry Tests
|
|
# =============================================================================
|
|
|
|
class TestSanityMain:
|
|
"""Tests for the CLI entry wrapper."""
|
|
|
|
def test_missing_directory_returns_one(self, capsys):
|
|
result = sanity_main(directory="/nonexistent/directory")
|
|
|
|
assert result == 1
|
|
assert "Error" in capsys.readouterr().err
|
|
|
|
def test_non_wiki_directory_returns_one(self, tmp_path, capsys):
|
|
empty = tmp_path / "empty"
|
|
empty.mkdir()
|
|
|
|
result = sanity_main(directory=str(empty))
|
|
|
|
assert result == 1
|
|
assert "Error" in capsys.readouterr().err
|
|
|
|
def test_forwards_non_interactive_to_wiki_sanity(self, minimal_wiki, monkeypatch):
|
|
received = {}
|
|
|
|
def fake_wiki_sanity(directory=".", json_output=False, non_interactive=False):
|
|
received["non_interactive"] = non_interactive
|
|
return 0
|
|
|
|
monkeypatch.setattr(sanity_manager, "wiki_sanity", fake_wiki_sanity)
|
|
|
|
result = sanity_main(directory=str(minimal_wiki), non_interactive=True)
|
|
|
|
assert result == 0
|
|
assert received["non_interactive"] is True
|
|
|
|
|
|
# =============================================================================
|
|
# Progress Display Tests (SPEC 7.14.2)
|
|
# =============================================================================
|
|
|
|
class TestWikiSanityProgress:
|
|
"""Tests for the in-run progress display wiring (SPEC 7.14.2)."""
|
|
|
|
@pytest.fixture
|
|
def fake_display_class(self, monkeypatch):
|
|
"""Replace run_sanity's ProgressDisplay with a recording stub."""
|
|
instances = []
|
|
|
|
class _StubDisplay:
|
|
def __init__(self, check_names, interactive, stream=None,
|
|
interval_seconds=3.0, expected_seconds=30.0,
|
|
clock=None, label="Running Sanity"):
|
|
self.check_names = list(check_names)
|
|
self.interactive = interactive
|
|
self.label = label
|
|
self.progress_calls = []
|
|
self.heartbeat_calls = []
|
|
self.finished = False
|
|
instances.append(self)
|
|
|
|
def progress(self, check_name, check_result):
|
|
self.progress_calls.append(check_name)
|
|
|
|
def heartbeat(self, remaining):
|
|
self.heartbeat_calls.append(list(remaining))
|
|
|
|
def finish(self):
|
|
self.finished = True
|
|
|
|
monkeypatch.setattr(run_sanity, "ProgressDisplay", _StubDisplay)
|
|
return instances
|
|
|
|
def test_display_covers_registered_checks(self, minimal_wiki, fake_display_class):
|
|
exit_code = wiki_sanity(str(minimal_wiki))
|
|
|
|
assert exit_code == 0
|
|
assert len(fake_display_class) == 1
|
|
display = fake_display_class[0]
|
|
assert display.check_names == [name for name, _check in sanity_checks()]
|
|
assert set(display.progress_calls) == {name for name, _check in sanity_checks()}
|
|
assert display.finished is True
|
|
|
|
def test_non_interactive_flag_drives_display_mode(
|
|
self, minimal_wiki, fake_display_class, monkeypatch
|
|
):
|
|
decisions = []
|
|
monkeypatch.setattr(
|
|
run_sanity, "should_use_interactive",
|
|
lambda flag, stream: decisions.append(flag) or flag,
|
|
)
|
|
|
|
wiki_sanity(str(minimal_wiki), non_interactive=True)
|
|
|
|
assert decisions == [True]
|
|
assert fake_display_class[0].interactive is True
|
|
|
|
def test_display_label_names_the_wiki_run(self, minimal_wiki, fake_display_class):
|
|
wiki_sanity(str(minimal_wiki))
|
|
|
|
assert fake_display_class[0].label == "Running Wiki Sanity"
|
|
|
|
def test_json_output_skips_display(self, minimal_wiki, fake_display_class, capsys):
|
|
exit_code = wiki_sanity(str(minimal_wiki), json_output=True)
|
|
|
|
assert exit_code == 0
|
|
assert fake_display_class == []
|
|
# The JSON stream stays pure: no progress noise around it
|
|
assert json.loads(capsys.readouterr().out)["passed"] is True
|