Files
fiona-assistant/tests/test_ingest.py
T
2026-10-01 07:04:55 +02:00

730 lines
28 KiB
Python

"""Tests for wiki ingest (SPEC 7.25).
Generated by AI/LLM: 2026-09-30 - acceptance checks for fetch, archive, convert, and write.
"""
from __future__ import annotations
import contextlib
import email.message
import hashlib
import io
import json
import pathlib
import shutil
import typing
import urllib.error
import urllib.request
import pypdf
import pypdf.generic
import pytest
import yaml
from fiona.customlinter.fmlint import FrontmatterLinter
from fiona.customlinter.mdlinter import MarkdownLinter
from fiona.paths import llm_wiki_template
from fiona.wiki.ingest.converters.base import SourceConverter
from fiona.wiki.ingest.fetchers import UrlFetcher
from fiona.wiki.ingest.media import slug_from_url
from fiona.wiki.ingest.models import ConversionResult, FetchedSource, IngestError
from fiona.wiki.ingest.pipeline import default_pipeline, ingest_main
from fiona.wiki.sanity_manager import run_wiki_checks
_FRONTMATTER_KEYS = (
"name",
"version",
"title",
"created",
"updated",
"type",
"status",
"sources",
"source_meta",
"ingest",
"tags",
"confidence",
"llm",
"contested",
"contradictions",
)
_INGEST_KEYS = (
"origin",
"origin_kind",
"final_url",
"fetched",
"media_type",
"size_bytes",
"sha256",
"raw_reused",
"converter",
"conversion",
"warnings",
"tool",
)
class _FakeResponse:
"""Minimal urllib response for one canned body."""
def __init__(self, body: bytes, header: str, url: str) -> None:
self._body = body
self._offset = 0
self.headers = email.message.Message()
self.headers["Content-Type"] = header
self._url = url
self.status = 200
def read(self, amount: int = -1) -> bytes:
"""Return the next chunk of the canned body."""
if amount is None or amount < 0:
amount = len(self._body) - self._offset
chunk = self._body[self._offset:self._offset + amount]
self._offset += len(chunk)
return chunk
def geturl(self) -> str:
"""Return the URL after redirects."""
return self._url
def __enter__(self) -> typing.Self:
return self
def __exit__(self, *_args: object) -> bool:
return False
class _FakeOpener:
"""URL opener that never touches the network."""
def __init__(self, body: bytes, header: str, *, status: int = 200, final_url: str | None = None) -> None:
self.body = body
self.header = header
self.status = status
self.final_url = final_url
self.requests: list[urllib.request.Request] = []
self.timeout: float | None = None
self.calls = 0
def open(self, request: urllib.request.Request, timeout: float | None = None) -> _FakeResponse:
"""Record the request and return the canned response or a 404."""
self.calls += 1
self.requests.append(request)
self.timeout = timeout
if self.status != 200:
raise urllib.error.HTTPError(
request.full_url,
self.status,
"Not Found",
email.message.Message(),
io.BytesIO(b""),
)
final = self.final_url or request.full_url
return _FakeResponse(self.body, self.header, final)
class _XTestConverter(SourceConverter):
"""Converter registered only by the test, for ``application/x-test``."""
name = "xtest"
media_types = ("application/x-test",)
extensions = (".xtest",)
raw_category = "articles"
def convert(self, source: FetchedSource) -> ConversionResult:
"""Return a fixed body so the test can see the registration."""
return ConversionResult(
body="xtest body\n",
title="X Test",
source_meta={},
conversion="complete",
warnings=[],
converter=self.name,
)
def _wiki(tmp_path: pathlib.Path) -> pathlib.Path:
"""Create a small llm-wiki with settings, a log, and an index."""
root = tmp_path / "wiki"
(root / "wiki").mkdir(parents=True)
(root / "ingest").mkdir()
(root / "raw").mkdir()
(root / "personal").mkdir()
shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json")
(root / "wiki" / "log.md").write_text(
"---\n"
"name: log\n"
"title: Wiki Log\n"
"version: 2.0.0\n"
"type: summary\n"
"created: 2026-09-30\n"
"updated: 2026-09-30\n"
"confidence: trusted\n"
"llm:\n"
" - model: none\n"
"contested: false\n"
"contradictions: []\n"
"---\n\n"
"# Wiki Log\n\n"
"- 2026-09-30 - Initialized the llm-wiki.\n",
encoding="utf-8",
newline="\n",
)
(root / "wiki" / "index.md").write_text(
"---\nname: index\nversion: 1.0.0\n---\n\n# Index\n",
encoding="utf-8",
newline="\n",
)
return root
def _run(root: pathlib.Path, *args: str) -> tuple[int, str, str]:
"""Run the ingest CLI against ``root`` and capture both streams."""
stdout = io.StringIO()
stderr = io.StringIO()
with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
code = ingest_main(["--wiki", str(root), *args])
return code, stdout.getvalue(), stderr.getvalue()
def _front(path: pathlib.Path) -> dict:
"""Parse page frontmatter with types left intact."""
text = path.read_text(encoding="utf-8")
return yaml.safe_load(text.split("---", 2)[1])
def _pdf(texts: list[str], *, title: str | None = None, password: str | None = None) -> bytes:
"""Build a PDF whose pages contain the given strings."""
writer = pypdf.PdfWriter()
for text in texts:
page = writer.add_blank_page(width=612, height=792)
if text:
_pdf_text(writer, page, text)
if title:
writer.add_metadata({"/Title": title})
if password:
writer.encrypt(password)
buffer = io.BytesIO()
writer.write(buffer)
return buffer.getvalue()
def _pdf_text(writer: pypdf.PdfWriter, page: pypdf.PageObject, text: str) -> None:
"""Draw one Helvetica string on a page."""
font = pypdf.generic.DictionaryObject({
pypdf.generic.NameObject("/Type"): pypdf.generic.NameObject("/Font"),
pypdf.generic.NameObject("/Subtype"): pypdf.generic.NameObject("/Type1"),
pypdf.generic.NameObject("/BaseFont"): pypdf.generic.NameObject("/Helvetica"),
})
font_ref = writer._add_object(font)
fonts = pypdf.generic.DictionaryObject({pypdf.generic.NameObject("/F1"): font_ref})
page[pypdf.generic.NameObject("/Resources")] = pypdf.generic.DictionaryObject({
pypdf.generic.NameObject("/Font"): fonts,
})
stream = pypdf.generic.DecodedStreamObject()
stream.set_data(f"BT /F1 12 Tf 72 100 Td ({text}) Tj ET".encode("latin-1"))
page[pypdf.generic.NameObject("/Contents")] = writer._add_object(stream)
def _hashes(root: pathlib.Path) -> dict[str, str]:
"""Hash every file under a wiki."""
found: dict[str, str] = {}
for path in root.rglob("*"):
if path.is_file():
found[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest()
return found
def test_slug_from_console_url() -> None:
"""SPEC 7.25.2.6: the last three segments become one hyphenated slug."""
assert slug_from_url("https://host/job/performance_TL153/2108/console") == (
"performance-tl153-2108-console"
)
def test_markdown_gets_frontmatter_and_keeps_body(tmp_path: pathlib.Path) -> None:
"""A markdown file keeps its wording. Missing page frontmatter is added."""
root = _wiki(tmp_path)
source = root / "ingest" / "Notes.md"
source.write_text("# Kept\n\nThe body stays.\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(source))
assert code == 0, stderr
assert not source.exists()
raw = root / "raw" / "articles" / "notes.md"
assert raw.read_text(encoding="utf-8") == "# Kept\n\nThe body stays.\n"
page = root / "wiki" / "sources" / "notes.md"
front = _front(page)
assert front["status"] == "raw"
assert front["source_meta"] == {}
assert "The body stays." in page.read_text(encoding="utf-8")
assert front["title"] == "Kept"
text = page.read_text(encoding="utf-8")
assert [line for line in text.splitlines() if line.startswith("# ")] == ["# Kept"]
def test_markdown_frontmatter_moves_to_source_meta(tmp_path: pathlib.Path) -> None:
"""Source frontmatter is metadata. The page still receives ingest frontmatter."""
root = _wiki(tmp_path)
source = tmp_path / "ready.md"
source.write_text(
"---\ntitle: Real Title\nauthor: Ada\n---\n\nParagraph.\n",
encoding="utf-8",
newline="\n",
)
code, _stdout, stderr = _run(root, str(source))
assert code == 0, stderr
page = root / "wiki" / "sources" / "ready.md"
front = _front(page)
assert front["title"] == "Real Title"
assert front["source_meta"]["author"] == "Ada"
assert "Paragraph." in page.read_text(encoding="utf-8")
assert source.read_text(encoding="utf-8").startswith("---\n")
def test_text_becomes_markdown_for_later_refinement(tmp_path: pathlib.Path) -> None:
"""A text file becomes a fenced page at status raw. The tool does not summarize it."""
root = _wiki(tmp_path)
source = tmp_path / "trace.log"
source.write_text("alpha\n```\nbeta\n", encoding="utf-8", newline="\n")
code, stdout, stderr = _run(root, str(source))
assert code == 0, stderr
page = root / "wiki" / "sources" / "trace.md"
front = _front(page)
assert front["status"] == "raw"
assert front["ingest"]["converter"] == "text"
assert front["ingest"]["conversion"] == "complete"
body = page.read_text(encoding="utf-8")
assert "````text" in body
assert "alpha" in body
assert "Not refined yet." in body
assert "Next: refine" in stdout
code, pending, _stderr = _run(root, "--pending")
assert code == 0
assert "wiki/sources/trace.md" in pending
def test_text_fallback_decode_warns(tmp_path: pathlib.Path) -> None:
"""Bytes that are not UTF-8 are decoded with a warning."""
root = _wiki(tmp_path)
source = tmp_path / "legacy.txt"
source.write_bytes(b"caf\x80")
code, _stdout, stderr = _run(root, str(source))
assert code == 0, stderr
front = _front(root / "wiki" / "sources" / "legacy.md")
assert any("cp1252" in warning for warning in front["ingest"]["warnings"])
def test_register_converter_without_pipeline_edits(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.1.2: a new format is one class and one registration."""
root = _wiki(tmp_path)
sample = tmp_path / "sample.xtest"
sample.write_bytes(b"hello")
pipeline = default_pipeline(root)
pipeline.registry.register(_XTestConverter)
result = pipeline.run(str(sample))
assert result.exit_code == 0
assert result.converter == "xtest"
assert (root / "raw" / "articles" / "sample.xtest").is_file()
assert "xtest body" in (root / "wiki" / "sources" / "sample.md").read_text(encoding="utf-8")
def test_ftp_scheme_writes_nothing(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.2.1: an unsupported scheme exits 1 and writes nothing."""
root = _wiki(tmp_path)
before = _hashes(root)
code, stdout, stderr = _run(root, "ftp://host/a.pdf")
assert code == 1
assert stdout == ""
assert "Unsupported URL scheme: ftp" in stderr
assert _hashes(root) == before
def test_missing_file_and_directory(tmp_path: pathlib.Path) -> None:
"""A missing path and a directory each exit 1."""
root = _wiki(tmp_path)
code, stdout, stderr = _run(root, str(tmp_path / "missing.txt"))
assert code == 1
assert stdout == ""
assert "Source not found:" in stderr
code, stdout, stderr = _run(root, str(tmp_path))
assert code == 1
assert stdout == ""
assert "Source is a directory:" in stderr
def test_download_uses_content_type_extension(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.2.5: a PDF response with no suffix is saved as ``.pdf``."""
root = _wiki(tmp_path)
body = _pdf(["Hello"])
opener = _FakeOpener(body, "application/pdf", final_url="https://host/report")
pipeline = default_pipeline(root)
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
result = pipeline.run("https://host/report")
assert result.exit_code == 0
assert result.raw_path == "raw/papers/report.pdf"
assert (root / "raw" / "papers" / "report.pdf").read_bytes() == body
assert not (root / "ingest" / "report.pdf").exists()
assert opener.requests[0].get_header("Authorization") is None
assert "fiona-assistant/" in opener.requests[0].get_header("User-agent")
def test_download_404_leaves_ingest_unchanged(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.2.3: a 404 exits 1 and does not add files."""
root = _wiki(tmp_path)
(root / "ingest" / "keep.txt").write_text("stay", encoding="utf-8")
before = sorted(path.name for path in (root / "ingest").iterdir())
opener = _FakeOpener(b"", "text/plain", status=404)
pipeline = default_pipeline(root)
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
with pytest.raises(IngestError, match="404"):
pipeline.run("https://host/missing")
assert sorted(path.name for path in (root / "ingest").iterdir()) == before
def test_download_over_limit_is_deleted(tmp_path: pathlib.Path) -> None:
"""A body over the limit exits 1 and leaves no partial file."""
root = _wiki(tmp_path)
pipeline = default_pipeline(root)
pipeline.settings.max_download_bytes = 4
opener = _FakeOpener(b"0123456789", "text/plain")
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
with pytest.raises(IngestError, match="exceeds 4 bytes"):
pipeline.run("https://host/big")
assert list((root / "ingest").iterdir()) == []
def test_extensionless_pdf_uses_pdf_converter(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.2.4: leading ``%PDF-`` bytes select the pdf converter."""
root = _wiki(tmp_path)
source = tmp_path / "notes"
source.write_bytes(_pdf(["Page text"]))
code, _stdout, stderr = _run(root, str(source))
assert code == 0, stderr
page = root / "wiki" / "sources" / "notes.md"
assert _front(page)["ingest"]["converter"] == "pdf"
assert (root / "raw" / "papers" / "notes.pdf").is_file()
def test_raw_override_and_escape(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.3.2: ``-r`` places the file, and ``..`` exits 1."""
root = _wiki(tmp_path)
html = tmp_path / "page.html"
html.write_text("<html><body><article><p>Hi</p></article></body></html>", encoding="utf-8")
code, _stdout, stderr = _run(root, str(html), "-r", "jenkins/tl153")
assert code == 0, stderr
assert (root / "raw" / "jenkins" / "tl153" / "page.html").is_file()
other = tmp_path / "other.html"
other.write_text("<html><body><p>Other</p></body></html>", encoding="utf-8")
before = _hashes(root)
code, stdout, stderr = _run(root, str(other), "-r", "../wiki/x")
assert code == 1
assert stdout == ""
assert "--raw must stay inside raw/" in stderr
assert _hashes(root) == before
def test_reuses_identical_bytes(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.3.3: a second name and ``-r`` reuse the first raw copy."""
root = _wiki(tmp_path)
first = tmp_path / "one.txt"
first.write_text("same bytes\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(first))
assert code == 0, stderr
second = tmp_path / "two.txt"
second.write_text("same bytes\n", encoding="utf-8", newline="\n")
pipeline = default_pipeline(root)
result = pipeline.run(str(second), raw="other/dir")
assert result.raw_reused is True
assert result.raw_path == "raw/articles/one.txt"
assert result.duplicate_page == "wiki/sources/one.md"
copies = [path for path in (root / "raw").rglob("*") if path.is_file() and path.suffix == ".txt"]
assert copies == [root / "raw" / "articles" / "one.txt"]
assert "already archived at raw/articles/one.txt; --raw ignored" in result.warnings
def test_changed_file_gets_a_second_raw_name(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.3.4: a changed file of the same name is stored as ``slug-2``."""
root = _wiki(tmp_path)
source = tmp_path / "notes.txt"
source.write_text("first\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(source), "--raw-only")
assert code == 0, stderr
original = (root / "raw" / "articles" / "notes.txt").read_bytes()
source.write_text("second\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(source))
assert code == 0, stderr
assert (root / "raw" / "articles" / "notes.txt").read_bytes() == original
assert (root / "raw" / "articles" / "notes-2.txt").read_text(encoding="utf-8") == "second\n"
def test_duplicate_exits_zero_and_skips_the_log(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.3.5: the same bytes report the first page and do not rewrite it."""
root = _wiki(tmp_path)
source = root / "ingest" / "a.txt"
source.write_text("alpha\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(source))
assert code == 0, stderr
page = root / "wiki" / "sources" / "a.md"
page_bytes = page.read_bytes()
log_before = (root / "wiki" / "log.md").read_text(encoding="utf-8")
again = root / "ingest" / "b.txt"
again.write_text("alpha\n", encoding="utf-8", newline="\n")
code, stdout, stderr = _run(root, str(again), "--json")
assert code == 0, stderr
payload = json.loads(stdout)
assert payload["duplicate_page"] == "wiki/sources/a.md"
assert payload["page_path"] is None
assert not again.exists()
assert page.read_bytes() == page_bytes
assert (root / "wiki" / "log.md").read_text(encoding="utf-8") == log_before
def test_ingest_drop_moves_and_personal_is_copied(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.3.6: ``ingest/`` is moved. A file outside it is copied."""
root = _wiki(tmp_path)
dropped = root / "ingest" / "a.pdf"
dropped.write_bytes(_pdf(["Moved"]))
code, _stdout, stderr = _run(root, str(dropped))
assert code == 0, stderr
assert not dropped.exists()
archived = root / "raw" / "papers" / "a.pdf"
assert archived.read_bytes() == _pdf(["Moved"])
personal = root / "personal" / "note.txt"
personal.write_text("keep me\n", encoding="utf-8", newline="\n")
original = personal.read_bytes()
code, _stdout, stderr = _run(root, str(personal))
assert code == 0, stderr
assert personal.read_bytes() == original
assert (root / "raw" / "articles" / "note.txt").read_bytes() == original
def test_unknown_bytes_archive_without_a_page(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.4.1: random bytes exit 3 under ``raw/other``."""
root = _wiki(tmp_path)
source = tmp_path / "blob.xyz"
source.write_bytes(b"\x00\x01\x02\xff")
code, stdout, _stderr = _run(root, str(source), "--json")
assert code == 3
payload = json.loads(stdout)
assert payload["conversion"] == "none"
assert payload["page_path"] is None
assert payload["raw_path"] == "raw/other/blob.xyz"
assert (root / "raw" / "other" / "blob.xyz").is_file()
assert not (root / "wiki" / "sources" / "blob.md").exists()
human_source = tmp_path / "other.xyz"
human_source.write_bytes(b"\x00\x02")
code, human, _stderr = _run(root, str(human_source))
assert code == 3
assert human.strip().splitlines()[-1].startswith("No converter for")
def test_encrypted_pdf_keeps_raw_and_skips_the_page(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.4.2: an encrypted PDF exits 3 and the raw copy stays."""
root = _wiki(tmp_path)
source = tmp_path / "secret.pdf"
source.write_bytes(_pdf(["hidden"], password="secret"))
code, stdout, stderr = _run(root, str(source))
assert code == 3
assert "Conversion failed: pdf:" in stderr
assert (root / "raw" / "papers" / "secret.pdf").is_file()
assert not (root / "wiki" / "sources" / "secret.md").exists()
assert stdout.strip().splitlines()[-1].startswith("No converter for")
def test_html_keeps_article_link_and_table(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.4.5: article text, no script, an absolute link, and a pipe table."""
root = _wiki(tmp_path)
html = """<!doctype html>
<html lang="en"><head><title>Page Title</title></head>
<body>
<nav>NAVSECRET</nav>
<article>
<p>ARTICLETEX</p>
<a href="/a">Go</a>
<table><tr><th>Col</th></tr><tr><td>Cell</td></tr></table>
</article>
<script>SCRIPTSECRET</script>
</body></html>
"""
opener = _FakeOpener(html.encode("utf-8"), "text/html")
pipeline = default_pipeline(root)
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
result = pipeline.run("https://host/p")
assert result.exit_code == 0
body = (root / result.page_path).read_text(encoding="utf-8")
assert "ARTICLETEX" in body
assert "SCRIPTSECRET" not in body
assert "https://host/a" in body
assert "|" in body
def test_pdf_page_headings_and_blank_page(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.4.6: pages are numbered, and a blank page is partial."""
root = _wiki(tmp_path)
numbered = tmp_path / "two.pdf"
numbered.write_bytes(_pdf(["Alpha", "Beta"]))
code, _stdout, stderr = _run(root, str(numbered))
assert code == 0, stderr
body = (root / "wiki" / "sources" / "two.md").read_text(encoding="utf-8")
assert "## Page 1" in body
assert "## Page 2" in body
blank = tmp_path / "gap.pdf"
blank.write_bytes(_pdf(["Alpha", ""]))
code, _stdout, stderr = _run(root, str(blank))
assert code == 0, stderr
front = _front(root / "wiki" / "sources" / "gap.md")
assert front["ingest"]["conversion"] == "partial"
assert "page 2 has no text layer" in front["ingest"]["warnings"]
def test_target_paths(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.5.1: ``-t concepts`` writes under concepts. Escapes exit 1."""
root = _wiki(tmp_path)
source = tmp_path / "idea.txt"
source.write_text("idea\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(source), "-t", "concepts")
assert code == 0, stderr
assert (root / "wiki" / "concepts" / "idea.md").is_file()
code, stdout, stderr = _run(root, str(source), "-t", "../raw/x.md")
assert code == 1
assert stdout == ""
assert "--target must stay inside wiki/" in stderr
other = tmp_path / "other.txt"
other.write_text("x\n", encoding="utf-8", newline="\n")
code, stdout, stderr = _run(root, str(other), "-t", "index.md")
assert code == 1
assert "--target cannot replace" in stderr
def test_existing_page_refuses_until_force(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.5.2: a second run exits 1. ``--force`` replaces the page."""
root = _wiki(tmp_path)
source = tmp_path / "item.txt"
source.write_text("one\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(source))
assert code == 0, stderr
page = root / "wiki" / "sources" / "item.md"
before = page.read_bytes()
opener = _FakeOpener(b"nope", "text/plain")
pipeline = default_pipeline(root)
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
with pytest.raises(IngestError, match="Page exists:"):
pipeline.run("https://host/item")
assert opener.calls == 0
assert page.read_bytes() == before
source.write_text("two\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(source), "--force")
assert code == 0, stderr
assert page.read_bytes() != before
assert "two" in (root / "raw" / "articles" / "item-2.txt").read_text(encoding="utf-8")
def test_page_frontmatter_matches_raw_hash(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.5.3 and 7.25.5.4: frontmatter is complete and the page has one H1."""
root = _wiki(tmp_path)
index_before = (root / "wiki" / "index.md").read_bytes()
source = tmp_path / "story.md"
source.write_text("# Story\n\nSee [gone](missing.md).\n", encoding="utf-8", newline="\n")
pipeline = default_pipeline(root)
result = pipeline.run(str(source))
assert result.exit_code == 0
page = root / result.page_path
front = _front(page)
for key in _FRONTMATTER_KEYS:
assert key in front
for key in _INGEST_KEYS:
assert key in front["ingest"]
raw = root / front["sources"][0]
assert front["ingest"]["sha256"] == hashlib.sha256(raw.read_bytes()).hexdigest()
assert front["ingest"]["warnings"] == result.warnings
assert front["status"] == "raw"
assert front["confidence"] == "zero"
assert front["llm"] == [{"model": "none"}]
headings = [line for line in page.read_text(encoding="utf-8").splitlines() if line.startswith("# ")]
assert headings == ["# Story"]
linter = FrontmatterLinter(settings_file=root / "fiona-settings.json")
assert linter.lint_file(page) == []
links = MarkdownLinter(settings_file=root / "fiona-settings.json")
assert [item for item in links.lint_file(page) if item.code == "MDL001"] == []
assert (root / "wiki" / "index.md").read_bytes() == index_before
log = (root / "wiki" / "log.md").read_text(encoding="utf-8")
assert "version: 2.0.1" in log
assert f"ingest {source.resolve()} -> raw/articles/story.md -> wiki/sources/story.md (raw)" in log
def test_style_violations_become_one_warning(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.5.5: violations the fixer leaves are counted in one warning."""
root = _wiki(tmp_path)
source = tmp_path / "long.md"
source.write_text("# Title\n\n" + ("word " * 40) + "\n", encoding="utf-8", newline="\n")
code, _stdout, stderr = _run(root, str(source))
assert code == 0, stderr
warnings = _front(root / "wiki" / "sources" / "long.md")["ingest"]["warnings"]
assert any(warning.endswith("markdown style violations left") for warning in warnings)
def test_raw_only_and_dry_run(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.7 and 7.25.8: raw-only skips the page. A dry run writes nothing."""
root = _wiki(tmp_path)
source = root / "ingest" / "only.txt"
source.write_text("archive me\n", encoding="utf-8", newline="\n")
code, stdout, stderr = _run(root, str(source), "--raw-only", "--json")
assert code == 0, stderr
payload = json.loads(stdout)
assert payload["conversion"] == "none"
assert payload["page_path"] is None
assert not source.exists()
assert (root / "raw" / "articles" / "only.txt").is_file()
kept = root / "ingest" / "stay.txt"
kept.write_text("stay\n", encoding="utf-8", newline="\n")
before = _hashes(root)
code, stdout, stderr = _run(root, str(kept), "--dry-run")
assert code == 0, stderr
assert stdout.startswith("Dry run - no files will be modified:")
assert kept.is_file()
assert _hashes(root) == before
code, stdout, _stderr = _run(root, str(kept), "--dry-run", "--json")
assert json.loads(stdout)["dry_run"] is True
def test_pending_hides_refined_pages(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.9: ``--pending`` lists raw pages and drops refined ones."""
root = _wiki(tmp_path)
source = tmp_path / "open.txt"
source.write_text("open\n", encoding="utf-8", newline="\n")
assert _run(root, str(source))[0] == 0
code, stdout, _stderr = _run(root, "--pending", "--json")
payload = json.loads(stdout)
assert code == 0
assert payload["total"] == 1
page = root / "wiki" / "sources" / "open.md"
text = page.read_text(encoding="utf-8").replace("status: raw", "status: refined", 1)
page.write_text(text, encoding="utf-8", newline="\n")
code, stdout, _stderr = _run(root, "--pending", "--json")
assert json.loads(stdout)["pending"] == []
def test_not_a_wiki_and_mutually_exclusive_args(tmp_path: pathlib.Path) -> None:
"""A directory without settings exits 1. Source and ``--pending`` cannot combine."""
code, stdout, stderr = _run(tmp_path, str(tmp_path / "a.txt"))
assert code == 1
assert stdout == ""
assert "is not an llm-wiki directory" in stderr
with pytest.raises(SystemExit) as caught:
ingest_main(["--pending", "file.txt"])
assert caught.value.code == 2
def test_sanity_ignores_raw_markdown(tmp_path: pathlib.Path) -> None:
"""SPEC 7.25.12: a raw markdown file with style violations does not fail sanity."""
root = _wiki(tmp_path)
raw = root / "raw" / "articles"
raw.mkdir(parents=True)
(raw / "bad-style.md").write_text(
"# Bad\n\n" + ("x" * 180) + "\n\n[gone](missing-file.md)\n",
encoding="utf-8",
newline="\n",
)
results = run_wiki_checks(root)
assert "bad-style.md" not in str(results)