Files
fiona-assistant/tests/test_ingest_edges.py
T

553 lines
21 KiB
Python

"""Branch coverage for wiki ingest stages.
Generated by AI/LLM: 2026-09-30 - error paths and converter edges from SPEC 7.1.5.
"""
from __future__ import annotations
import contextlib
import datetime
import email.message
import importlib.metadata
import io
import json
import pathlib
import shutil
import types
import typing
import urllib.error
import urllib.request
import pypdf
import pytest
from pymarkdown.api import PyMarkdownApiException
from fiona.mdoctor.mdoctor import MDoctor
from fiona.paths import llm_wiki_template
from fiona.wiki.ingest import log_writer as log_mod
from fiona.wiki.ingest import media
from fiona.wiki.ingest import page_writer as pages
from fiona.wiki.ingest import settings as settings_mod
from fiona.wiki.ingest.archiver import RawArchiver, _strip_prefix
from fiona.wiki.ingest.converters import default_registry
from fiona.wiki.ingest.converters.base import ConversionError, ConverterRegistry
from fiona.wiki.ingest.converters.html import HtmlConverter
from fiona.wiki.ingest.converters.markdown import MarkdownConverter
from fiona.wiki.ingest.converters.pdf import (
PdfConverter,
_iso_date,
_metadata,
_outline,
_render,
_walk_outline,
)
from fiona.wiki.ingest.fetchers import UrlFetcher, _without_userinfo, url_scheme
from fiona.wiki.ingest.log_writer import IngestLog
from fiona.wiki.ingest.models import ArchiveResult, ConversionResult, FetchedSource, IngestError
from fiona.wiki.ingest.pipeline import (
IngestPipeline,
_citing_page,
_print_pending,
default_pipeline,
)
from fiona.wiki.ingest.settings import default_settings, load_ingest_settings
def _wiki(tmp_path: pathlib.Path) -> pathlib.Path:
"""Create a small llm-wiki."""
root = tmp_path / "wiki"
(root / "wiki").mkdir(parents=True)
(root / "ingest").mkdir()
(root / "raw").mkdir()
shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json")
(root / "wiki" / "log.md").write_text(
"---\nname: log\nversion: 2.0.0\nupdated: 2026-09-30\n---\n\n# Log\n",
encoding="utf-8",
newline="\n",
)
(root / "wiki" / "index.md").write_text(
"---\nname: index\nversion: 1.0.0\n---\n\n# Index\n",
encoding="utf-8",
newline="\n",
)
return root
def _source(path: pathlib.Path, *, origin_kind: str = "file", origin: str | None = None) -> FetchedSource:
"""Build a fetched-source record for a file that already exists."""
return FetchedSource(
argument=str(path),
path=path,
origin=origin or str(path.resolve()),
origin_kind=origin_kind,
final_url=None,
media_type="text/plain",
charset=None,
size_bytes=path.stat().st_size,
slug=path.stem,
fetched="2026-09-30T00:00:00Z",
created=False,
)
class _FakeResponse:
"""One canned download body."""
def __init__(self, body: bytes, url: str) -> None:
self._body = body
self._offset = 0
self.headers = email.message.Message()
self.headers["Content-Type"] = "text/plain"
self._url = url
def read(self, amount: int = -1) -> bytes:
"""Return the next slice of the body."""
if amount < 0:
amount = len(self._body) - self._offset
chunk = self._body[self._offset:self._offset + amount]
self._offset += len(chunk)
return chunk
def geturl(self) -> str:
"""Return the final URL."""
return self._url
def __enter__(self) -> typing.Self:
return self
def __exit__(self, *_args: object) -> bool:
return False
class _Opener:
"""Opener that returns one body or raises URLError."""
def __init__(self, body: bytes | None = b"hello\n", *, error: str | None = None) -> None:
self.body = body or b""
self.error = error
def open(self, request: urllib.request.Request, timeout: float | None = None) -> _FakeResponse:
"""Return the canned body or fail the download."""
del timeout
if self.error:
raise urllib.error.URLError(self.error)
return _FakeResponse(self.body, request.full_url)
def test_slug_and_media_edges() -> None:
"""Slugs, sniffing, extensions, and decode fallbacks."""
assert url_scheme("note.txt") is None
assert media.slug_from_url("https://host/index") == "host"
assert media.slug_from_url("https://host/") == "host"
assert media.split_content_type("text/plain; boundary; charset=\"utf-8\"") == ("text/plain", "utf-8")
assert media.split_content_type("application/octet-stream") == (None, None)
registry = default_registry()
assert media.sniff_media_type(b" <!DOCTYPE html><html>") == "text/html"
assert media.sniff_media_type(b"\x00\xff") == "application/octet-stream"
assert media.sniff_media_type(b"ab\xe2\x82") == "text/plain"
assert media.sniff_media_type(b"plain text") == "text/plain"
assert media.extension_for("application/zip", "pdf", registry) == ".pdf"
assert media.extension_for("application/zip", "", registry) == ".bin"
assert media.decode_bytes(b"\xef\xbb\xbfhi", None) == ("hi", [])
text, warnings = media.decode_bytes(b"\xff", "latin-1")
assert text == "ÿ"
assert warnings == ["decoded as latin-1; not valid UTF-8"]
_text, fallback = media.decode_bytes(b"\xff", "not-a-charset")
assert "cp1252" in fallback[0]
assert registry.find("nope", "md").name == "markdown"
def test_settings_fallbacks(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Missing, broken, and partial ingest settings keep usable values."""
def missing(_name: str) -> str:
raise importlib.metadata.PackageNotFoundError
monkeypatch.setattr(settings_mod.importlib.metadata, "version", missing)
assert settings_mod.package_version() == "0.99.0"
assert load_ingest_settings(tmp_path).timeout_seconds == 30
broken = tmp_path / "bad"
broken.mkdir()
(broken / "fiona-settings.json").write_text("{", encoding="utf-8")
assert load_ingest_settings(broken).max_download_bytes == 52428800
good = tmp_path / "good"
good.mkdir()
(good / "fiona-settings.json").write_text(json.dumps({
"fiona": {"wiki": {"ingest": {
"user_agent": "agent",
"default_target": "concepts",
"raw_categories": {"pdf": "papers", "html": 1},
}}},
}), encoding="utf-8")
loaded = load_ingest_settings(good)
assert loaded.user_agent == "agent"
assert loaded.default_target == "concepts"
assert loaded.raw_categories == {"pdf": "papers"}
def test_markdown_frontmatter_edges(tmp_path: pathlib.Path) -> None:
"""Invalid, empty, and non-mapping frontmatter, and a fenced heading."""
converter = MarkdownConverter()
cases = {
"bad.md": "---\n[\n---\n# Kept\n",
"empty.md": "---\n\n---\n# Kept\n",
"list.md": "---\n- a\n---\n# Kept\n",
"fenced.md": "```\n# Hidden\n```\n",
}
for name, text in cases.items():
path = tmp_path / name
path.write_text(text, encoding="utf-8")
result = converter.convert(_source(path))
assert result.conversion == "complete"
fenced = converter.convert(_source(tmp_path / "fenced.md"))
assert fenced.title == "fenced"
assert fenced.body.startswith("```")
assert converter.convert(_source(tmp_path / "bad.md")).warnings
def test_html_edges(tmp_path: pathlib.Path) -> None:
"""Comments, empty pages, og:title, images, and meta tags."""
converter = HtmlConverter()
html = """<!DOCTYPE html>
<html lang="en"><head>
<base href="https://host/base/">
<meta property="og:title" content="">
<meta name="author" content="Ada">
<meta property="og:site_name" content="Site">
<meta property="article:published_time" content="2020-01-01">
<meta name="unused" content="no">
<meta name="description">
<link rel="canonical" href="https://host/c">
<!-- secret -->
</head><body>
<h1> </h1>
<article><h1>Real</h1><img src="pic.png" alt="pic"></article>
</body></html>
"""
path = tmp_path / "page.html"
path.write_text(html, encoding="utf-8")
source = _source(path, origin_kind="url", origin="https://host/p")
result = converter.convert(source)
assert "secret" not in result.body
assert "https://host/base/pic.png" in result.body
assert result.source_meta["author"] == "Ada"
assert result.source_meta["canonical"] == "https://host/c"
assert result.title == "Real"
empty = tmp_path / "empty.html"
empty.write_text("<html><body><script>only</script></body></html>", encoding="utf-8")
partial = converter.convert(_source(empty, origin_kind="url", origin="https://host/e"))
assert partial.conversion == "partial"
og = tmp_path / "og.html"
og.write_text(
'<html><head><meta property="og:title" content="Og"></head><body><p>x</p></body></html>',
encoding="utf-8",
)
assert converter.convert(_source(og)).title == "Og"
bare = tmp_path / "bare.html"
bare.write_text("<html><body><p>x</p></body></html>", encoding="utf-8")
assert converter.convert(_source(bare)).title is None
blank_h1 = tmp_path / "blankh1.html"
blank_h1.write_text("<html><body><h1> </h1><p>x</p></body></html>", encoding="utf-8")
assert converter.convert(_source(blank_h1)).title is None
def test_pdf_metadata_outline_and_failures(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""PDF dates, outline headings, a scanned file, and read failures."""
aware = datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC)
naive = datetime.datetime(2020, 1, 1) # noqa: DTZ001
assert _iso_date(None) is None
assert _iso_date(naive).startswith("2020-01-01T")
assert _iso_date(aware).startswith("2020-01-02T")
assert _iso_date("D:20200103") == "2020-01-03T00:00:00Z"
assert _iso_date("nope") == "nope"
assert _iso_date("") is None
info_obj = types.SimpleNamespace(
title="T",
author="",
subject="S",
keywords=None,
creation_date=naive,
modification_date=aware,
)
empty = types.SimpleNamespace(pages=[1], metadata=None)
info_reader = types.SimpleNamespace(pages=[1, 2], metadata=info_obj)
assert _metadata(empty) == {"page_count": 1} # type: ignore[arg-type]
info = _metadata(info_reader) # type: ignore[arg-type]
assert info["title"] == "T"
assert "author" not in info
class _OutlineReader:
"""Fake reader whose first outline item has no destination."""
def get_destination_page_number(self, item: object) -> int:
if getattr(item, "title", None) == "":
raise ValueError("bad")
return 0
found: list[tuple[int, int, str]] = []
bad = types.SimpleNamespace(title="")
good = types.SimpleNamespace(title=None)
_walk_outline(_OutlineReader(), [bad, [good]], 5, found) # type: ignore[arg-type]
assert found == [(0, 6, "Untitled")]
assert _render(["body"], [(0, 2, "Chapter")]) == "## Chapter\n\nbody\n"
assert "## Page" not in _render([""], [(1, 2, "Later")])
class _BrokenOutline:
"""Fake reader whose outline cannot be read."""
@property
def outline(self) -> list[object]:
raise pypdf.errors.PdfReadError("no outline")
assert _outline(_BrokenOutline()) == [] # type: ignore[arg-type]
writer = pypdf.PdfWriter()
writer.add_blank_page(width=200, height=200)
parent = writer.add_outline_item("Parent", 0)
writer.add_outline_item("Child", 0, parent=parent)
writer.add_metadata({"/Title": "Outlined", "/CreationDate": "D:20200101120000"})
buffer = io.BytesIO()
writer.write(buffer)
outlined = tmp_path / "outlined.pdf"
outlined.write_bytes(buffer.getvalue())
converted = PdfConverter().convert(_source(outlined))
assert "Parent" in converted.body
assert converted.title == "Outlined"
blank = tmp_path / "blank.pdf"
blank_writer = pypdf.PdfWriter()
blank_writer.add_blank_page(width=200, height=200)
blank_buffer = io.BytesIO()
blank_writer.write(blank_buffer)
blank.write_bytes(blank_buffer.getvalue())
scanned = PdfConverter().convert(_source(blank))
assert scanned.conversion == "partial"
assert any("scanned" in warning for warning in scanned.warnings)
corrupt = tmp_path / "corrupt.pdf"
corrupt.write_bytes(b"%PDF-1.7 not a real file")
with pytest.raises(ConversionError):
PdfConverter().convert(_source(corrupt))
def extract_text() -> str:
raise pypdf.errors.PdfReadError("page")
reader = types.SimpleNamespace(
is_encrypted=False,
metadata=None,
pages=[types.SimpleNamespace(extract_text=extract_text)],
outline=[],
)
monkeypatch.setattr(pypdf, "PdfReader", lambda _path: reader)
with pytest.raises(ConversionError, match="page"):
PdfConverter().convert(_source(outlined))
def test_archive_edges(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Reuse skips, raw escapes, named raw files, and collision suffixes."""
root = _wiki(tmp_path)
archiver = RawArchiver(tmp_path / "empty")
assert archiver.find_by_sha256("ab" * 32, 1) is None
assert _strip_prefix("raw", "raw") == ""
assert _strip_prefix("raw/notes/a.txt", "raw") == "notes/a.txt"
registry = default_registry()
settings = default_settings()
already = root / "raw" / "articles" / "already.md"
already.parent.mkdir(parents=True)
already.write_text("# Already\n", encoding="utf-8")
placed = archiver_on(root).archive(
_source(already),
registry=registry,
settings=settings,
raw_override=None,
dry_run=False,
)
assert placed.raw_reused is False
with pytest.raises(IngestError, match="already under raw"):
archiver_on(root).archive(
_source(already),
registry=registry,
settings=settings,
raw_override="other",
dry_run=False,
)
original_stat = pathlib.Path.stat
seen: dict[str, int] = {}
def flaky(self: pathlib.Path, *args: object, **kwargs: object) -> object:
if self.name == "bad.bin":
seen[str(self)] = seen.get(str(self), 0) + 1
if seen[str(self)] > 1:
raise OSError("stat failed")
return original_stat(self, *args, **kwargs)
(root / "raw" / "bad.bin").write_bytes(b"x")
monkeypatch.setattr(pathlib.Path, "stat", flaky)
assert archiver_on(root).find_by_sha256("ab" * 32, 1) is None
monkeypatch.undo()
pipeline = default_pipeline(root)
named = tmp_path / "fresh.txt"
named.write_text("fresh\n", encoding="utf-8")
pipeline.run(str(named), raw="notes/kept.html", raw_only=True)
assert (root / "raw" / "notes" / "kept.html").is_file()
for index, text in enumerate(("one\n", "two\n", "three\n"), start=1):
path = tmp_path / f"dir{index}" / "note.txt"
path.parent.mkdir()
path.write_text(text, encoding="utf-8")
pipeline.run(str(path), raw_only=True)
assert (root / "raw" / "articles" / "note-3.txt").is_file()
def archiver_on(root: pathlib.Path) -> RawArchiver:
"""Return an archiver bound to ``root``."""
return RawArchiver(root)
def test_download_and_log_edges(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""URL errors, a failed save, userinfo, and a log that cannot be bumped."""
root = _wiki(tmp_path)
registry = default_registry()
fetcher = UrlFetcher(registry, opener=_Opener(error="timed out"))
with pytest.raises(IngestError, match="timed out"):
fetcher.fetch("https://host/a.txt", root / "ingest")
assert _without_userinfo("https://user:secret@host:443/a.txt") == "https://host:443/a.txt"
assert _without_userinfo("https://host/a.txt") == "https://host/a.txt"
original_write = pathlib.Path.write_bytes
def fail_write(self: pathlib.Path, data: bytes) -> int:
original_write(self, data)
raise OSError("disk")
monkeypatch.setattr(pathlib.Path, "write_bytes", fail_write)
fetcher = UrlFetcher(registry, opener=_Opener(b"hello\n"))
with pytest.raises(OSError, match="disk"):
fetcher.fetch("https://host/b.txt", root / "ingest")
monkeypatch.undo()
assert not list((root / "ingest").glob("*.partial"))
log = IngestLog(root)
log.append(day="2026-09-30", origin="o", raw_rel="raw/a", page_rel=None, dry_run=True)
(root / "wiki" / "log.md").unlink()
log.append(day="2026-09-30", origin="o", raw_rel="raw/a", page_rel="wiki/a.md", dry_run=False)
odd = root / "wiki" / "log.md"
odd.write_text("version: beta\n# Log", encoding="utf-8", newline="\n")
log.append(day="2026-09-30", origin="o", raw_rel="raw/a", page_rel=None, dry_run=False)
text = odd.read_text(encoding="utf-8")
assert "version: beta" in text
assert text.endswith("\n")
assert log_mod._set_updated("no updated here\n", "2026-09-30") == "no updated here\n"
assert log_mod._bump_version("version: 1\n") == "version: 1\n"
assert log_mod._bump_version("no version here\n") == "no version here\n"
def test_page_writer_edges(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""Heading fences, links, fixer failures, and target prefixes."""
body = "# Title\n\n```\n# inside\n```\n\n###### Six\n"
shifted = pages._shift_headings(body)
assert "```\n# inside\n```" in shifted
assert "###### Six" in shifted
assert pages._keeps_target("?q", tmp_path) is True
assert pages._keeps_target("missing.md", tmp_path) is False
assert pages._body_after_frontmatter("no front\n") == "no front\n"
assert pages._body_after_frontmatter("---\nonly\n") == "---\nonly\n"
assert pages._split_ending("a\r\n") == ("\r\n", "a")
assert pages._split_ending("a") == ("", "a")
assert pages._strip_wiki_prefix("wiki") == ""
assert pages._strip_wiki_prefix("wiki/concepts") == "concepts"
root = _wiki(tmp_path)
writer = pages.RawPageWriter(root)
source_path = tmp_path / "note.txt"
source_path.write_text("note\n", encoding="utf-8")
source = _source(source_path)
archive_path = root / "raw" / "articles" / "note.txt"
archive_path.parent.mkdir(parents=True, exist_ok=True)
archive_path.write_text("note\n", encoding="utf-8")
archive = ArchiveResult(archive_path, "raw/articles/note.txt", "ab" * 32, False, [])
conversion = ConversionResult("plain\n", "Note", {}, "complete", [], "text")
def broken_fix(self: MDoctor, path: str) -> None:
raise PyMarkdownApiException("fix")
def broken_scan(self: MDoctor, path: str) -> list:
raise PyMarkdownApiException("scan")
monkeypatch.setattr(MDoctor, "fix_file_programmatically", broken_fix)
monkeypatch.setattr(MDoctor, "scan_file_programmatically", broken_scan)
monkeypatch.setattr(pages, "_body_after_frontmatter", lambda _text: "no heading\n")
page = root / "wiki" / "sources" / "note.md"
writer.write(
source=source,
archive=archive,
conversion=conversion,
page=page,
dry_run=False,
)
assert page.is_file()
assert page.read_text(encoding="utf-8").count("# Note") == 1
def test_pipeline_edges(tmp_path: pathlib.Path) -> None:
"""Dry-run downloads, pending pages, and a forced duplicate."""
root = _wiki(tmp_path)
pipeline = default_pipeline(root)
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(b"hello\n"))
result = pipeline.run("https://user:secret@host/hello.txt", dry_run=True)
assert result.dry_run is True
assert list((root / "ingest").iterdir()) == []
with pytest.raises(IngestError, match="stay inside raw"):
pipeline.run("https://host/escape.txt", raw="../outside")
assert list((root / "ingest").glob("*.txt")) == []
bare = IngestPipeline(
root,
[],
ConverterRegistry(),
pipeline.archiver,
pipeline.page_writer,
pipeline.log_writer,
default_settings(),
)
local = tmp_path / "local.txt"
local.write_text("local\n", encoding="utf-8")
with pytest.raises(IngestError, match="Source not found"):
bare.run(str(local))
(root / "wiki" / "skip.md").mkdir()
(root / "wiki" / "bad.md").write_text("---\n[\n---\n", encoding="utf-8")
(root / "wiki" / "sources").mkdir(exist_ok=True)
(root / "wiki" / "sources" / "stringy.md").write_text(
"---\nname: stringy\nstatus: raw\nsources: raw/a.txt\ningest: plain\n---\n\n# S\n",
encoding="utf-8",
newline="\n",
)
pending = pipeline.list_pending()
assert any(item["page_path"].endswith("stringy.md") for item in pending)
assert _citing_page(root, "raw/a.txt", "nope") == "wiki/sources/stringy.md"
captured = io_capture(_print_pending, pipeline, False)
assert "stringy.md" in captured
shutil.rmtree(root / "wiki")
assert pipeline.list_pending() == []
assert _citing_page(root, "raw/a.txt", "nope") is None
root = _wiki(tmp_path / "second")
pipeline = default_pipeline(root)
first = tmp_path / "first.txt"
second = tmp_path / "other" / "second.txt"
first.write_text("same\n", encoding="utf-8")
second.parent.mkdir()
second.write_text("same\n", encoding="utf-8")
pipeline.run(str(first))
forced = pipeline.run(str(second), force=True)
assert forced.page_path == "wiki/sources/first.md"
def io_capture(func: typing.Callable, *args: object) -> str:
"""Return stdout from a function that prints."""
buffer = io.StringIO()
with contextlib.redirect_stdout(buffer):
func(*args)
return buffer.getvalue()