553 lines
21 KiB
Python
553 lines
21 KiB
Python
"""Branch coverage for wiki ingest stages.
|
|
|
|
Generated by AI/LLM: 2026-09-30 - error paths and converter edges from SPEC 7.1.5.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import datetime
|
|
import email.message
|
|
import importlib.metadata
|
|
import io
|
|
import json
|
|
import pathlib
|
|
import shutil
|
|
import types
|
|
import typing
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
import pypdf
|
|
import pytest
|
|
from pymarkdown.api import PyMarkdownApiException
|
|
|
|
from fiona.mdoctor.mdoctor import MDoctor
|
|
from fiona.paths import llm_wiki_template
|
|
from fiona.wiki.ingest import log_writer as log_mod
|
|
from fiona.wiki.ingest import media
|
|
from fiona.wiki.ingest import page_writer as pages
|
|
from fiona.wiki.ingest import settings as settings_mod
|
|
from fiona.wiki.ingest.archiver import RawArchiver, _strip_prefix
|
|
from fiona.wiki.ingest.converters import default_registry
|
|
from fiona.wiki.ingest.converters.base import ConversionError, ConverterRegistry
|
|
from fiona.wiki.ingest.converters.html import HtmlConverter
|
|
from fiona.wiki.ingest.converters.markdown import MarkdownConverter
|
|
from fiona.wiki.ingest.converters.pdf import (
|
|
PdfConverter,
|
|
_iso_date,
|
|
_metadata,
|
|
_outline,
|
|
_render,
|
|
_walk_outline,
|
|
)
|
|
from fiona.wiki.ingest.fetchers import UrlFetcher, _without_userinfo, url_scheme
|
|
from fiona.wiki.ingest.log_writer import IngestLog
|
|
from fiona.wiki.ingest.models import ArchiveResult, ConversionResult, FetchedSource, IngestError
|
|
from fiona.wiki.ingest.pipeline import (
|
|
IngestPipeline,
|
|
_citing_page,
|
|
_print_pending,
|
|
default_pipeline,
|
|
)
|
|
from fiona.wiki.ingest.settings import default_settings, load_ingest_settings
|
|
|
|
|
|
def _wiki(tmp_path: pathlib.Path) -> pathlib.Path:
|
|
"""Create a small llm-wiki."""
|
|
root = tmp_path / "wiki"
|
|
(root / "wiki").mkdir(parents=True)
|
|
(root / "ingest").mkdir()
|
|
(root / "raw").mkdir()
|
|
shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json")
|
|
(root / "wiki" / "log.md").write_text(
|
|
"---\nname: log\nversion: 2.0.0\nupdated: 2026-09-30\n---\n\n# Log\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
(root / "wiki" / "index.md").write_text(
|
|
"---\nname: index\nversion: 1.0.0\n---\n\n# Index\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
return root
|
|
|
|
|
|
def _source(path: pathlib.Path, *, origin_kind: str = "file", origin: str | None = None) -> FetchedSource:
|
|
"""Build a fetched-source record for a file that already exists."""
|
|
return FetchedSource(
|
|
argument=str(path),
|
|
path=path,
|
|
origin=origin or str(path.resolve()),
|
|
origin_kind=origin_kind,
|
|
final_url=None,
|
|
media_type="text/plain",
|
|
charset=None,
|
|
size_bytes=path.stat().st_size,
|
|
slug=path.stem,
|
|
fetched="2026-09-30T00:00:00Z",
|
|
created=False,
|
|
)
|
|
|
|
|
|
class _FakeResponse:
|
|
"""One canned download body."""
|
|
|
|
def __init__(self, body: bytes, url: str) -> None:
|
|
self._body = body
|
|
self._offset = 0
|
|
self.headers = email.message.Message()
|
|
self.headers["Content-Type"] = "text/plain"
|
|
self._url = url
|
|
|
|
def read(self, amount: int = -1) -> bytes:
|
|
"""Return the next slice of the body."""
|
|
if amount < 0:
|
|
amount = len(self._body) - self._offset
|
|
chunk = self._body[self._offset:self._offset + amount]
|
|
self._offset += len(chunk)
|
|
return chunk
|
|
|
|
def geturl(self) -> str:
|
|
"""Return the final URL."""
|
|
return self._url
|
|
|
|
def __enter__(self) -> typing.Self:
|
|
return self
|
|
|
|
def __exit__(self, *_args: object) -> bool:
|
|
return False
|
|
|
|
|
|
class _Opener:
|
|
"""Opener that returns one body or raises URLError."""
|
|
|
|
def __init__(self, body: bytes | None = b"hello\n", *, error: str | None = None) -> None:
|
|
self.body = body or b""
|
|
self.error = error
|
|
|
|
def open(self, request: urllib.request.Request, timeout: float | None = None) -> _FakeResponse:
|
|
"""Return the canned body or fail the download."""
|
|
del timeout
|
|
if self.error:
|
|
raise urllib.error.URLError(self.error)
|
|
return _FakeResponse(self.body, request.full_url)
|
|
|
|
|
|
def test_slug_and_media_edges() -> None:
|
|
"""Slugs, sniffing, extensions, and decode fallbacks."""
|
|
assert url_scheme("note.txt") is None
|
|
assert media.slug_from_url("https://host/index") == "host"
|
|
assert media.slug_from_url("https://host/") == "host"
|
|
assert media.split_content_type("text/plain; boundary; charset=\"utf-8\"") == ("text/plain", "utf-8")
|
|
assert media.split_content_type("application/octet-stream") == (None, None)
|
|
registry = default_registry()
|
|
assert media.sniff_media_type(b" <!DOCTYPE html><html>") == "text/html"
|
|
assert media.sniff_media_type(b"\x00\xff") == "application/octet-stream"
|
|
assert media.sniff_media_type(b"ab\xe2\x82") == "text/plain"
|
|
assert media.sniff_media_type(b"plain text") == "text/plain"
|
|
assert media.extension_for("application/zip", "pdf", registry) == ".pdf"
|
|
assert media.extension_for("application/zip", "", registry) == ".bin"
|
|
assert media.decode_bytes(b"\xef\xbb\xbfhi", None) == ("hi", [])
|
|
text, warnings = media.decode_bytes(b"\xff", "latin-1")
|
|
assert text == "ÿ"
|
|
assert warnings == ["decoded as latin-1; not valid UTF-8"]
|
|
_text, fallback = media.decode_bytes(b"\xff", "not-a-charset")
|
|
assert "cp1252" in fallback[0]
|
|
assert registry.find("nope", "md").name == "markdown"
|
|
|
|
|
|
def test_settings_fallbacks(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Missing, broken, and partial ingest settings keep usable values."""
|
|
def missing(_name: str) -> str:
|
|
raise importlib.metadata.PackageNotFoundError
|
|
|
|
monkeypatch.setattr(settings_mod.importlib.metadata, "version", missing)
|
|
assert settings_mod.package_version() == "0.99.0"
|
|
assert load_ingest_settings(tmp_path).timeout_seconds == 30
|
|
broken = tmp_path / "bad"
|
|
broken.mkdir()
|
|
(broken / "fiona-settings.json").write_text("{", encoding="utf-8")
|
|
assert load_ingest_settings(broken).max_download_bytes == 52428800
|
|
good = tmp_path / "good"
|
|
good.mkdir()
|
|
(good / "fiona-settings.json").write_text(json.dumps({
|
|
"fiona": {"wiki": {"ingest": {
|
|
"user_agent": "agent",
|
|
"default_target": "concepts",
|
|
"raw_categories": {"pdf": "papers", "html": 1},
|
|
}}},
|
|
}), encoding="utf-8")
|
|
loaded = load_ingest_settings(good)
|
|
assert loaded.user_agent == "agent"
|
|
assert loaded.default_target == "concepts"
|
|
assert loaded.raw_categories == {"pdf": "papers"}
|
|
|
|
|
|
def test_markdown_frontmatter_edges(tmp_path: pathlib.Path) -> None:
|
|
"""Invalid, empty, and non-mapping frontmatter, and a fenced heading."""
|
|
converter = MarkdownConverter()
|
|
cases = {
|
|
"bad.md": "---\n[\n---\n# Kept\n",
|
|
"empty.md": "---\n\n---\n# Kept\n",
|
|
"list.md": "---\n- a\n---\n# Kept\n",
|
|
"fenced.md": "```\n# Hidden\n```\n",
|
|
}
|
|
for name, text in cases.items():
|
|
path = tmp_path / name
|
|
path.write_text(text, encoding="utf-8")
|
|
result = converter.convert(_source(path))
|
|
assert result.conversion == "complete"
|
|
fenced = converter.convert(_source(tmp_path / "fenced.md"))
|
|
assert fenced.title == "fenced"
|
|
assert fenced.body.startswith("```")
|
|
assert converter.convert(_source(tmp_path / "bad.md")).warnings
|
|
|
|
|
|
def test_html_edges(tmp_path: pathlib.Path) -> None:
|
|
"""Comments, empty pages, og:title, images, and meta tags."""
|
|
converter = HtmlConverter()
|
|
html = """<!DOCTYPE html>
|
|
<html lang="en"><head>
|
|
<base href="https://host/base/">
|
|
<meta property="og:title" content="">
|
|
<meta name="author" content="Ada">
|
|
<meta property="og:site_name" content="Site">
|
|
<meta property="article:published_time" content="2020-01-01">
|
|
<meta name="unused" content="no">
|
|
<meta name="description">
|
|
<link rel="canonical" href="https://host/c">
|
|
<!-- secret -->
|
|
</head><body>
|
|
<h1> </h1>
|
|
<article><h1>Real</h1><img src="pic.png" alt="pic"></article>
|
|
</body></html>
|
|
"""
|
|
path = tmp_path / "page.html"
|
|
path.write_text(html, encoding="utf-8")
|
|
source = _source(path, origin_kind="url", origin="https://host/p")
|
|
result = converter.convert(source)
|
|
assert "secret" not in result.body
|
|
assert "https://host/base/pic.png" in result.body
|
|
assert result.source_meta["author"] == "Ada"
|
|
assert result.source_meta["canonical"] == "https://host/c"
|
|
assert result.title == "Real"
|
|
empty = tmp_path / "empty.html"
|
|
empty.write_text("<html><body><script>only</script></body></html>", encoding="utf-8")
|
|
partial = converter.convert(_source(empty, origin_kind="url", origin="https://host/e"))
|
|
assert partial.conversion == "partial"
|
|
og = tmp_path / "og.html"
|
|
og.write_text(
|
|
'<html><head><meta property="og:title" content="Og"></head><body><p>x</p></body></html>',
|
|
encoding="utf-8",
|
|
)
|
|
assert converter.convert(_source(og)).title == "Og"
|
|
bare = tmp_path / "bare.html"
|
|
bare.write_text("<html><body><p>x</p></body></html>", encoding="utf-8")
|
|
assert converter.convert(_source(bare)).title is None
|
|
blank_h1 = tmp_path / "blankh1.html"
|
|
blank_h1.write_text("<html><body><h1> </h1><p>x</p></body></html>", encoding="utf-8")
|
|
assert converter.convert(_source(blank_h1)).title is None
|
|
|
|
|
|
def test_pdf_metadata_outline_and_failures(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""PDF dates, outline headings, a scanned file, and read failures."""
|
|
aware = datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC)
|
|
naive = datetime.datetime(2020, 1, 1) # noqa: DTZ001
|
|
assert _iso_date(None) is None
|
|
assert _iso_date(naive).startswith("2020-01-01T")
|
|
assert _iso_date(aware).startswith("2020-01-02T")
|
|
assert _iso_date("D:20200103") == "2020-01-03T00:00:00Z"
|
|
assert _iso_date("nope") == "nope"
|
|
assert _iso_date("") is None
|
|
|
|
info_obj = types.SimpleNamespace(
|
|
title="T",
|
|
author="",
|
|
subject="S",
|
|
keywords=None,
|
|
creation_date=naive,
|
|
modification_date=aware,
|
|
)
|
|
empty = types.SimpleNamespace(pages=[1], metadata=None)
|
|
info_reader = types.SimpleNamespace(pages=[1, 2], metadata=info_obj)
|
|
assert _metadata(empty) == {"page_count": 1} # type: ignore[arg-type]
|
|
info = _metadata(info_reader) # type: ignore[arg-type]
|
|
assert info["title"] == "T"
|
|
assert "author" not in info
|
|
|
|
class _OutlineReader:
|
|
"""Fake reader whose first outline item has no destination."""
|
|
|
|
def get_destination_page_number(self, item: object) -> int:
|
|
if getattr(item, "title", None) == "":
|
|
raise ValueError("bad")
|
|
return 0
|
|
|
|
found: list[tuple[int, int, str]] = []
|
|
bad = types.SimpleNamespace(title="")
|
|
good = types.SimpleNamespace(title=None)
|
|
_walk_outline(_OutlineReader(), [bad, [good]], 5, found) # type: ignore[arg-type]
|
|
assert found == [(0, 6, "Untitled")]
|
|
assert _render(["body"], [(0, 2, "Chapter")]) == "## Chapter\n\nbody\n"
|
|
assert "## Page" not in _render([""], [(1, 2, "Later")])
|
|
|
|
class _BrokenOutline:
|
|
"""Fake reader whose outline cannot be read."""
|
|
|
|
@property
|
|
def outline(self) -> list[object]:
|
|
raise pypdf.errors.PdfReadError("no outline")
|
|
|
|
assert _outline(_BrokenOutline()) == [] # type: ignore[arg-type]
|
|
|
|
writer = pypdf.PdfWriter()
|
|
writer.add_blank_page(width=200, height=200)
|
|
parent = writer.add_outline_item("Parent", 0)
|
|
writer.add_outline_item("Child", 0, parent=parent)
|
|
writer.add_metadata({"/Title": "Outlined", "/CreationDate": "D:20200101120000"})
|
|
buffer = io.BytesIO()
|
|
writer.write(buffer)
|
|
outlined = tmp_path / "outlined.pdf"
|
|
outlined.write_bytes(buffer.getvalue())
|
|
converted = PdfConverter().convert(_source(outlined))
|
|
assert "Parent" in converted.body
|
|
assert converted.title == "Outlined"
|
|
|
|
blank = tmp_path / "blank.pdf"
|
|
blank_writer = pypdf.PdfWriter()
|
|
blank_writer.add_blank_page(width=200, height=200)
|
|
blank_buffer = io.BytesIO()
|
|
blank_writer.write(blank_buffer)
|
|
blank.write_bytes(blank_buffer.getvalue())
|
|
scanned = PdfConverter().convert(_source(blank))
|
|
assert scanned.conversion == "partial"
|
|
assert any("scanned" in warning for warning in scanned.warnings)
|
|
|
|
corrupt = tmp_path / "corrupt.pdf"
|
|
corrupt.write_bytes(b"%PDF-1.7 not a real file")
|
|
with pytest.raises(ConversionError):
|
|
PdfConverter().convert(_source(corrupt))
|
|
|
|
def extract_text() -> str:
|
|
raise pypdf.errors.PdfReadError("page")
|
|
|
|
reader = types.SimpleNamespace(
|
|
is_encrypted=False,
|
|
metadata=None,
|
|
pages=[types.SimpleNamespace(extract_text=extract_text)],
|
|
outline=[],
|
|
)
|
|
monkeypatch.setattr(pypdf, "PdfReader", lambda _path: reader)
|
|
with pytest.raises(ConversionError, match="page"):
|
|
PdfConverter().convert(_source(outlined))
|
|
|
|
|
|
def test_archive_edges(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Reuse skips, raw escapes, named raw files, and collision suffixes."""
|
|
root = _wiki(tmp_path)
|
|
archiver = RawArchiver(tmp_path / "empty")
|
|
assert archiver.find_by_sha256("ab" * 32, 1) is None
|
|
assert _strip_prefix("raw", "raw") == ""
|
|
assert _strip_prefix("raw/notes/a.txt", "raw") == "notes/a.txt"
|
|
registry = default_registry()
|
|
settings = default_settings()
|
|
already = root / "raw" / "articles" / "already.md"
|
|
already.parent.mkdir(parents=True)
|
|
already.write_text("# Already\n", encoding="utf-8")
|
|
placed = archiver_on(root).archive(
|
|
_source(already),
|
|
registry=registry,
|
|
settings=settings,
|
|
raw_override=None,
|
|
dry_run=False,
|
|
)
|
|
assert placed.raw_reused is False
|
|
with pytest.raises(IngestError, match="already under raw"):
|
|
archiver_on(root).archive(
|
|
_source(already),
|
|
registry=registry,
|
|
settings=settings,
|
|
raw_override="other",
|
|
dry_run=False,
|
|
)
|
|
original_stat = pathlib.Path.stat
|
|
seen: dict[str, int] = {}
|
|
|
|
def flaky(self: pathlib.Path, *args: object, **kwargs: object) -> object:
|
|
if self.name == "bad.bin":
|
|
seen[str(self)] = seen.get(str(self), 0) + 1
|
|
if seen[str(self)] > 1:
|
|
raise OSError("stat failed")
|
|
return original_stat(self, *args, **kwargs)
|
|
|
|
(root / "raw" / "bad.bin").write_bytes(b"x")
|
|
monkeypatch.setattr(pathlib.Path, "stat", flaky)
|
|
assert archiver_on(root).find_by_sha256("ab" * 32, 1) is None
|
|
monkeypatch.undo()
|
|
|
|
pipeline = default_pipeline(root)
|
|
named = tmp_path / "fresh.txt"
|
|
named.write_text("fresh\n", encoding="utf-8")
|
|
pipeline.run(str(named), raw="notes/kept.html", raw_only=True)
|
|
assert (root / "raw" / "notes" / "kept.html").is_file()
|
|
for index, text in enumerate(("one\n", "two\n", "three\n"), start=1):
|
|
path = tmp_path / f"dir{index}" / "note.txt"
|
|
path.parent.mkdir()
|
|
path.write_text(text, encoding="utf-8")
|
|
pipeline.run(str(path), raw_only=True)
|
|
assert (root / "raw" / "articles" / "note-3.txt").is_file()
|
|
|
|
|
|
def archiver_on(root: pathlib.Path) -> RawArchiver:
|
|
"""Return an archiver bound to ``root``."""
|
|
return RawArchiver(root)
|
|
|
|
|
|
def test_download_and_log_edges(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""URL errors, a failed save, userinfo, and a log that cannot be bumped."""
|
|
root = _wiki(tmp_path)
|
|
registry = default_registry()
|
|
fetcher = UrlFetcher(registry, opener=_Opener(error="timed out"))
|
|
with pytest.raises(IngestError, match="timed out"):
|
|
fetcher.fetch("https://host/a.txt", root / "ingest")
|
|
assert _without_userinfo("https://user:secret@host:443/a.txt") == "https://host:443/a.txt"
|
|
assert _without_userinfo("https://host/a.txt") == "https://host/a.txt"
|
|
|
|
original_write = pathlib.Path.write_bytes
|
|
|
|
def fail_write(self: pathlib.Path, data: bytes) -> int:
|
|
original_write(self, data)
|
|
raise OSError("disk")
|
|
|
|
monkeypatch.setattr(pathlib.Path, "write_bytes", fail_write)
|
|
fetcher = UrlFetcher(registry, opener=_Opener(b"hello\n"))
|
|
with pytest.raises(OSError, match="disk"):
|
|
fetcher.fetch("https://host/b.txt", root / "ingest")
|
|
monkeypatch.undo()
|
|
assert not list((root / "ingest").glob("*.partial"))
|
|
|
|
log = IngestLog(root)
|
|
log.append(day="2026-09-30", origin="o", raw_rel="raw/a", page_rel=None, dry_run=True)
|
|
(root / "wiki" / "log.md").unlink()
|
|
log.append(day="2026-09-30", origin="o", raw_rel="raw/a", page_rel="wiki/a.md", dry_run=False)
|
|
odd = root / "wiki" / "log.md"
|
|
odd.write_text("version: beta\n# Log", encoding="utf-8", newline="\n")
|
|
log.append(day="2026-09-30", origin="o", raw_rel="raw/a", page_rel=None, dry_run=False)
|
|
text = odd.read_text(encoding="utf-8")
|
|
assert "version: beta" in text
|
|
assert text.endswith("\n")
|
|
assert log_mod._set_updated("no updated here\n", "2026-09-30") == "no updated here\n"
|
|
assert log_mod._bump_version("version: 1\n") == "version: 1\n"
|
|
assert log_mod._bump_version("no version here\n") == "no version here\n"
|
|
|
|
|
|
def test_page_writer_edges(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Heading fences, links, fixer failures, and target prefixes."""
|
|
body = "# Title\n\n```\n# inside\n```\n\n###### Six\n"
|
|
shifted = pages._shift_headings(body)
|
|
assert "```\n# inside\n```" in shifted
|
|
assert "###### Six" in shifted
|
|
assert pages._keeps_target("?q", tmp_path) is True
|
|
assert pages._keeps_target("missing.md", tmp_path) is False
|
|
assert pages._body_after_frontmatter("no front\n") == "no front\n"
|
|
assert pages._body_after_frontmatter("---\nonly\n") == "---\nonly\n"
|
|
assert pages._split_ending("a\r\n") == ("\r\n", "a")
|
|
assert pages._split_ending("a") == ("", "a")
|
|
assert pages._strip_wiki_prefix("wiki") == ""
|
|
assert pages._strip_wiki_prefix("wiki/concepts") == "concepts"
|
|
|
|
root = _wiki(tmp_path)
|
|
writer = pages.RawPageWriter(root)
|
|
source_path = tmp_path / "note.txt"
|
|
source_path.write_text("note\n", encoding="utf-8")
|
|
source = _source(source_path)
|
|
archive_path = root / "raw" / "articles" / "note.txt"
|
|
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
archive_path.write_text("note\n", encoding="utf-8")
|
|
archive = ArchiveResult(archive_path, "raw/articles/note.txt", "ab" * 32, False, [])
|
|
conversion = ConversionResult("plain\n", "Note", {}, "complete", [], "text")
|
|
|
|
def broken_fix(self: MDoctor, path: str) -> None:
|
|
raise PyMarkdownApiException("fix")
|
|
|
|
def broken_scan(self: MDoctor, path: str) -> list:
|
|
raise PyMarkdownApiException("scan")
|
|
|
|
monkeypatch.setattr(MDoctor, "fix_file_programmatically", broken_fix)
|
|
monkeypatch.setattr(MDoctor, "scan_file_programmatically", broken_scan)
|
|
monkeypatch.setattr(pages, "_body_after_frontmatter", lambda _text: "no heading\n")
|
|
page = root / "wiki" / "sources" / "note.md"
|
|
writer.write(
|
|
source=source,
|
|
archive=archive,
|
|
conversion=conversion,
|
|
page=page,
|
|
dry_run=False,
|
|
)
|
|
assert page.is_file()
|
|
assert page.read_text(encoding="utf-8").count("# Note") == 1
|
|
|
|
|
|
def test_pipeline_edges(tmp_path: pathlib.Path) -> None:
|
|
"""Dry-run downloads, pending pages, and a forced duplicate."""
|
|
root = _wiki(tmp_path)
|
|
pipeline = default_pipeline(root)
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(b"hello\n"))
|
|
result = pipeline.run("https://user:secret@host/hello.txt", dry_run=True)
|
|
assert result.dry_run is True
|
|
assert list((root / "ingest").iterdir()) == []
|
|
with pytest.raises(IngestError, match="stay inside raw"):
|
|
pipeline.run("https://host/escape.txt", raw="../outside")
|
|
assert list((root / "ingest").glob("*.txt")) == []
|
|
|
|
bare = IngestPipeline(
|
|
root,
|
|
[],
|
|
ConverterRegistry(),
|
|
pipeline.archiver,
|
|
pipeline.page_writer,
|
|
pipeline.log_writer,
|
|
default_settings(),
|
|
)
|
|
local = tmp_path / "local.txt"
|
|
local.write_text("local\n", encoding="utf-8")
|
|
with pytest.raises(IngestError, match="Source not found"):
|
|
bare.run(str(local))
|
|
|
|
(root / "wiki" / "skip.md").mkdir()
|
|
(root / "wiki" / "bad.md").write_text("---\n[\n---\n", encoding="utf-8")
|
|
(root / "wiki" / "sources").mkdir(exist_ok=True)
|
|
(root / "wiki" / "sources" / "stringy.md").write_text(
|
|
"---\nname: stringy\nstatus: raw\nsources: raw/a.txt\ningest: plain\n---\n\n# S\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
pending = pipeline.list_pending()
|
|
assert any(item["page_path"].endswith("stringy.md") for item in pending)
|
|
assert _citing_page(root, "raw/a.txt", "nope") == "wiki/sources/stringy.md"
|
|
captured = io_capture(_print_pending, pipeline, False)
|
|
assert "stringy.md" in captured
|
|
shutil.rmtree(root / "wiki")
|
|
assert pipeline.list_pending() == []
|
|
assert _citing_page(root, "raw/a.txt", "nope") is None
|
|
|
|
root = _wiki(tmp_path / "second")
|
|
pipeline = default_pipeline(root)
|
|
first = tmp_path / "first.txt"
|
|
second = tmp_path / "other" / "second.txt"
|
|
first.write_text("same\n", encoding="utf-8")
|
|
second.parent.mkdir()
|
|
second.write_text("same\n", encoding="utf-8")
|
|
pipeline.run(str(first))
|
|
forced = pipeline.run(str(second), force=True)
|
|
assert forced.page_path == "wiki/sources/first.md"
|
|
|
|
|
|
def io_capture(func: typing.Callable, *args: object) -> str:
|
|
"""Return stdout from a function that prints."""
|
|
buffer = io.StringIO()
|
|
with contextlib.redirect_stdout(buffer):
|
|
func(*args)
|
|
return buffer.getvalue()
|