de28441943
MEMORY.md and skill memory template, ingest auth and html_sweep, TOC slug fix, run_tests CI output, and matching tests. Excludes GitLab CI and publishing docs.
750 lines
29 KiB
Python
750 lines
29 KiB
Python
"""Tests for wiki ingest (SPEC 7.1.5).
|
|
|
|
Generated by AI/LLM: 2026-09-30 - acceptance checks for fetch, archive, convert, and write.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import email.message
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import pathlib
|
|
import shutil
|
|
import typing
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
import pypdf
|
|
import pypdf.generic
|
|
import pytest
|
|
import yaml
|
|
|
|
from fiona.customlinter.fmlint import FrontmatterLinter
|
|
from fiona.customlinter.mdlinter import MarkdownLinter
|
|
from fiona.paths import llm_wiki_template
|
|
from fiona.wiki.ingest.converters.base import SourceConverter
|
|
from fiona.wiki.ingest.fetchers import UrlFetcher
|
|
from fiona.wiki.ingest.media import slug_from_url
|
|
from fiona.wiki.ingest.models import ConversionResult, FetchedSource, IngestError
|
|
from fiona.wiki.ingest.pipeline import default_pipeline, ingest_main
|
|
from fiona.wiki.sanity_manager import run_wiki_checks
|
|
|
|
_FRONTMATTER_KEYS = (
|
|
"name",
|
|
"version",
|
|
"title",
|
|
"created",
|
|
"updated",
|
|
"type",
|
|
"status",
|
|
"sources",
|
|
"source_meta",
|
|
"ingest",
|
|
"tags",
|
|
"confidence",
|
|
"llm",
|
|
"contested",
|
|
"contradictions",
|
|
)
|
|
_INGEST_KEYS = (
|
|
"origin",
|
|
"origin_kind",
|
|
"final_url",
|
|
"fetched",
|
|
"media_type",
|
|
"size_bytes",
|
|
"sha256",
|
|
"raw_reused",
|
|
"converter",
|
|
"conversion",
|
|
"warnings",
|
|
"tool",
|
|
)
|
|
|
|
|
|
class _FakeResponse:
|
|
"""Minimal urllib response for one canned body."""
|
|
|
|
def __init__(self, body: bytes, header: str, url: str) -> None:
|
|
self._body = body
|
|
self._offset = 0
|
|
self.headers = email.message.Message()
|
|
self.headers["Content-Type"] = header
|
|
self._url = url
|
|
self.status = 200
|
|
|
|
def read(self, amount: int = -1) -> bytes:
|
|
"""Return the next chunk of the canned body."""
|
|
if amount is None or amount < 0:
|
|
amount = len(self._body) - self._offset
|
|
chunk = self._body[self._offset:self._offset + amount]
|
|
self._offset += len(chunk)
|
|
return chunk
|
|
|
|
def geturl(self) -> str:
|
|
"""Return the URL after redirects."""
|
|
return self._url
|
|
|
|
def __enter__(self) -> typing.Self:
|
|
return self
|
|
|
|
def __exit__(self, *_args: object) -> bool:
|
|
return False
|
|
|
|
|
|
class _FakeOpener:
|
|
"""URL opener that never touches the network."""
|
|
|
|
def __init__(self, body: bytes, header: str, *, status: int = 200, final_url: str | None = None) -> None:
|
|
self.body = body
|
|
self.header = header
|
|
self.status = status
|
|
self.final_url = final_url
|
|
self.requests: list[urllib.request.Request] = []
|
|
self.timeout: float | None = None
|
|
self.calls = 0
|
|
|
|
def open(self, request: urllib.request.Request, timeout: float | None = None) -> _FakeResponse:
|
|
"""Record the request and return the canned response or a 404."""
|
|
self.calls += 1
|
|
self.requests.append(request)
|
|
self.timeout = timeout
|
|
if self.status != 200:
|
|
raise urllib.error.HTTPError(
|
|
request.full_url,
|
|
self.status,
|
|
"Not Found",
|
|
email.message.Message(),
|
|
io.BytesIO(b""),
|
|
)
|
|
final = self.final_url or request.full_url
|
|
return _FakeResponse(self.body, self.header, final)
|
|
|
|
|
|
class _XTestConverter(SourceConverter):
|
|
"""Converter registered only by the test, for ``application/x-test``."""
|
|
|
|
name = "xtest"
|
|
media_types = ("application/x-test",)
|
|
extensions = (".xtest",)
|
|
raw_category = "articles"
|
|
|
|
def convert(self, source: FetchedSource) -> ConversionResult:
|
|
"""Return a fixed body so the test can see the registration."""
|
|
return ConversionResult(
|
|
body="xtest body\n",
|
|
title="X Test",
|
|
source_meta={},
|
|
conversion="complete",
|
|
warnings=[],
|
|
converter=self.name,
|
|
)
|
|
|
|
|
|
def _wiki(tmp_path: pathlib.Path) -> pathlib.Path:
|
|
"""Create a small llm-wiki with settings, a log, and an index."""
|
|
root = tmp_path / "wiki"
|
|
(root / "wiki").mkdir(parents=True)
|
|
(root / "ingest").mkdir()
|
|
(root / "raw").mkdir()
|
|
(root / "personal").mkdir()
|
|
shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json")
|
|
(root / "wiki" / "log.md").write_text(
|
|
"---\n"
|
|
"name: log\n"
|
|
"title: Wiki Log\n"
|
|
"version: 2.0.0\n"
|
|
"type: summary\n"
|
|
"created: 2026-09-30\n"
|
|
"updated: 2026-09-30\n"
|
|
"confidence: trusted\n"
|
|
"llm:\n"
|
|
" - model: none\n"
|
|
"contested: false\n"
|
|
"contradictions: []\n"
|
|
"---\n\n"
|
|
"# Wiki Log\n\n"
|
|
"- 2026-09-30 - Initialized the llm-wiki.\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
(root / "wiki" / "index.md").write_text(
|
|
"---\nname: index\nversion: 1.0.0\n---\n\n# Index\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
return root
|
|
|
|
|
|
def _run(root: pathlib.Path, *args: str) -> tuple[int, str, str]:
|
|
"""Run the ingest CLI against ``root`` and capture both streams."""
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
|
|
code = ingest_main(["--wiki", str(root), *args])
|
|
return code, stdout.getvalue(), stderr.getvalue()
|
|
|
|
|
|
def _front(path: pathlib.Path) -> dict:
|
|
"""Parse page frontmatter with types left intact."""
|
|
text = path.read_text(encoding="utf-8")
|
|
return yaml.safe_load(text.split("---", 2)[1])
|
|
|
|
|
|
def _pdf(texts: list[str], *, title: str | None = None, password: str | None = None) -> bytes:
|
|
"""Build a PDF whose pages contain the given strings."""
|
|
writer = pypdf.PdfWriter()
|
|
for text in texts:
|
|
page = writer.add_blank_page(width=612, height=792)
|
|
if text:
|
|
_pdf_text(writer, page, text)
|
|
if title:
|
|
writer.add_metadata({"/Title": title})
|
|
if password:
|
|
writer.encrypt(password)
|
|
buffer = io.BytesIO()
|
|
writer.write(buffer)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def _pdf_text(writer: pypdf.PdfWriter, page: pypdf.PageObject, text: str) -> None:
|
|
"""Draw one Helvetica string on a page."""
|
|
font = pypdf.generic.DictionaryObject({
|
|
pypdf.generic.NameObject("/Type"): pypdf.generic.NameObject("/Font"),
|
|
pypdf.generic.NameObject("/Subtype"): pypdf.generic.NameObject("/Type1"),
|
|
pypdf.generic.NameObject("/BaseFont"): pypdf.generic.NameObject("/Helvetica"),
|
|
})
|
|
font_ref = writer._add_object(font)
|
|
fonts = pypdf.generic.DictionaryObject({pypdf.generic.NameObject("/F1"): font_ref})
|
|
page[pypdf.generic.NameObject("/Resources")] = pypdf.generic.DictionaryObject({
|
|
pypdf.generic.NameObject("/Font"): fonts,
|
|
})
|
|
stream = pypdf.generic.DecodedStreamObject()
|
|
stream.set_data(f"BT /F1 12 Tf 72 100 Td ({text}) Tj ET".encode("latin-1"))
|
|
page[pypdf.generic.NameObject("/Contents")] = writer._add_object(stream)
|
|
|
|
|
|
def _hashes(root: pathlib.Path) -> dict[str, str]:
|
|
"""Hash every file under a wiki."""
|
|
found: dict[str, str] = {}
|
|
for path in root.rglob("*"):
|
|
if path.is_file():
|
|
found[path.relative_to(root).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
return found
|
|
|
|
|
|
def test_slug_from_console_url() -> None:
|
|
"""SPEC 7.1.5.2.6: the last three segments become one hyphenated slug."""
|
|
assert slug_from_url("https://host/job/performance_TL153/2108/console") == (
|
|
"performance-tl153-2108-console"
|
|
)
|
|
|
|
|
|
def test_markdown_gets_frontmatter_and_keeps_body(tmp_path: pathlib.Path) -> None:
|
|
"""A markdown file keeps its wording. Missing page frontmatter is added."""
|
|
root = _wiki(tmp_path)
|
|
source = root / "ingest" / "Notes.md"
|
|
source.write_text("# Kept\n\nThe body stays.\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
assert not source.exists()
|
|
raw = root / "raw" / "articles" / "notes.md"
|
|
assert raw.read_text(encoding="utf-8") == "# Kept\n\nThe body stays.\n"
|
|
page = root / "wiki" / "sources" / "notes.md"
|
|
front = _front(page)
|
|
assert front["status"] == "raw"
|
|
assert front["source_meta"] == {}
|
|
assert "The body stays." in page.read_text(encoding="utf-8")
|
|
assert front["title"] == "Kept"
|
|
text = page.read_text(encoding="utf-8")
|
|
assert [line for line in text.splitlines() if line.startswith("# ")] == ["# Kept"]
|
|
|
|
|
|
def test_markdown_frontmatter_moves_to_source_meta(tmp_path: pathlib.Path) -> None:
|
|
"""Source frontmatter is metadata. The page still receives ingest frontmatter."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "ready.md"
|
|
source.write_text(
|
|
"---\ntitle: Real Title\nauthor: Ada\n---\n\nParagraph.\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
code, _stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
page = root / "wiki" / "sources" / "ready.md"
|
|
front = _front(page)
|
|
assert front["title"] == "Real Title"
|
|
assert front["source_meta"]["author"] == "Ada"
|
|
assert "Paragraph." in page.read_text(encoding="utf-8")
|
|
assert source.read_text(encoding="utf-8").startswith("---\n")
|
|
|
|
|
|
def test_text_becomes_markdown_for_later_refinement(tmp_path: pathlib.Path) -> None:
|
|
"""A text file becomes a fenced page at status raw. The tool does not summarize it."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "trace.log"
|
|
source.write_text("alpha\n```\nbeta\n", encoding="utf-8", newline="\n")
|
|
code, stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
page = root / "wiki" / "sources" / "trace.md"
|
|
front = _front(page)
|
|
assert front["status"] == "raw"
|
|
assert front["ingest"]["converter"] == "text"
|
|
assert front["ingest"]["conversion"] == "complete"
|
|
body = page.read_text(encoding="utf-8")
|
|
assert "````text" in body
|
|
assert "alpha" in body
|
|
assert "Not refined yet." in body
|
|
assert "Next: refine" in stdout
|
|
code, pending, _stderr = _run(root, "--pending")
|
|
assert code == 0
|
|
assert "wiki/sources/trace.md" in pending
|
|
|
|
|
|
def test_text_fallback_decode_warns(tmp_path: pathlib.Path) -> None:
|
|
"""Bytes that are not UTF-8 are decoded with a warning."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "legacy.txt"
|
|
source.write_bytes(b"caf\x80")
|
|
code, _stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
front = _front(root / "wiki" / "sources" / "legacy.md")
|
|
assert any("cp1252" in warning for warning in front["ingest"]["warnings"])
|
|
|
|
|
|
def test_register_converter_without_pipeline_edits(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.1.2: a new format is one class and one registration."""
|
|
root = _wiki(tmp_path)
|
|
sample = tmp_path / "sample.xtest"
|
|
sample.write_bytes(b"hello")
|
|
pipeline = default_pipeline(root)
|
|
pipeline.registry.register(_XTestConverter)
|
|
result = pipeline.run(str(sample))
|
|
assert result.exit_code == 0
|
|
assert result.converter == "xtest"
|
|
assert (root / "raw" / "articles" / "sample.xtest").is_file()
|
|
assert "xtest body" in (root / "wiki" / "sources" / "sample.md").read_text(encoding="utf-8")
|
|
|
|
|
|
def test_ftp_scheme_writes_nothing(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.2.1: an unsupported scheme exits 1 and writes nothing."""
|
|
root = _wiki(tmp_path)
|
|
before = _hashes(root)
|
|
code, stdout, stderr = _run(root, "ftp://host/a.pdf")
|
|
assert code == 1
|
|
assert stdout == ""
|
|
assert "Unsupported URL scheme: ftp" in stderr
|
|
assert _hashes(root) == before
|
|
|
|
|
|
def test_missing_file_and_directory(tmp_path: pathlib.Path) -> None:
|
|
"""A missing path and a directory each exit 1."""
|
|
root = _wiki(tmp_path)
|
|
code, stdout, stderr = _run(root, str(tmp_path / "missing.txt"))
|
|
assert code == 1
|
|
assert stdout == ""
|
|
assert "Source not found:" in stderr
|
|
code, stdout, stderr = _run(root, str(tmp_path))
|
|
assert code == 1
|
|
assert stdout == ""
|
|
assert "Source is a directory:" in stderr
|
|
|
|
|
|
def test_download_uses_content_type_extension(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.2.5: a PDF response with no suffix is saved as ``.pdf``."""
|
|
root = _wiki(tmp_path)
|
|
body = _pdf(["Hello"])
|
|
opener = _FakeOpener(body, "application/pdf", final_url="https://host/report")
|
|
pipeline = default_pipeline(root)
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
|
|
result = pipeline.run("https://host/report")
|
|
assert result.exit_code == 0
|
|
assert result.raw_path == "raw/papers/report.pdf"
|
|
assert (root / "raw" / "papers" / "report.pdf").read_bytes() == body
|
|
assert not (root / "ingest" / "report.pdf").exists()
|
|
assert opener.requests[0].get_header("Authorization") is None
|
|
assert "fiona-assistant/" in opener.requests[0].get_header("User-agent")
|
|
|
|
|
|
def test_download_404_leaves_ingest_unchanged(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.2.3: a 404 exits 1 and does not add files."""
|
|
root = _wiki(tmp_path)
|
|
(root / "ingest" / "keep.txt").write_text("stay", encoding="utf-8")
|
|
before = sorted(path.name for path in (root / "ingest").iterdir())
|
|
opener = _FakeOpener(b"", "text/plain", status=404)
|
|
pipeline = default_pipeline(root)
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
|
|
with pytest.raises(IngestError, match="404"):
|
|
pipeline.run("https://host/missing")
|
|
assert sorted(path.name for path in (root / "ingest").iterdir()) == before
|
|
|
|
|
|
def test_download_over_limit_is_deleted(tmp_path: pathlib.Path) -> None:
|
|
"""A body over the limit exits 1 and leaves no partial file."""
|
|
root = _wiki(tmp_path)
|
|
pipeline = default_pipeline(root)
|
|
pipeline.settings.max_download_bytes = 4
|
|
opener = _FakeOpener(b"0123456789", "text/plain")
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
|
|
with pytest.raises(IngestError, match="exceeds 4 bytes"):
|
|
pipeline.run("https://host/big")
|
|
assert list((root / "ingest").iterdir()) == []
|
|
|
|
|
|
def test_extensionless_pdf_uses_pdf_converter(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.2.4: leading ``%PDF-`` bytes select the pdf converter."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "notes"
|
|
source.write_bytes(_pdf(["Page text"]))
|
|
code, _stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
page = root / "wiki" / "sources" / "notes.md"
|
|
assert _front(page)["ingest"]["converter"] == "pdf"
|
|
assert (root / "raw" / "papers" / "notes.pdf").is_file()
|
|
|
|
|
|
def test_raw_override_and_escape(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.3.2: ``-r`` places the file, and ``..`` exits 1."""
|
|
root = _wiki(tmp_path)
|
|
html = tmp_path / "page.html"
|
|
html.write_text("<html><body><article><p>Hi</p></article></body></html>", encoding="utf-8")
|
|
code, _stdout, stderr = _run(root, str(html), "-r", "jenkins/tl153")
|
|
assert code == 0, stderr
|
|
assert (root / "raw" / "jenkins" / "tl153" / "page.html").is_file()
|
|
other = tmp_path / "other.html"
|
|
other.write_text("<html><body><p>Other</p></body></html>", encoding="utf-8")
|
|
before = _hashes(root)
|
|
code, stdout, stderr = _run(root, str(other), "-r", "../wiki/x")
|
|
assert code == 1
|
|
assert stdout == ""
|
|
assert "--raw must stay inside raw/" in stderr
|
|
assert _hashes(root) == before
|
|
|
|
|
|
def test_reuses_identical_bytes(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.3.3: a second name and ``-r`` reuse the first raw copy."""
|
|
root = _wiki(tmp_path)
|
|
first = tmp_path / "one.txt"
|
|
first.write_text("same bytes\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(first))
|
|
assert code == 0, stderr
|
|
second = tmp_path / "two.txt"
|
|
second.write_text("same bytes\n", encoding="utf-8", newline="\n")
|
|
pipeline = default_pipeline(root)
|
|
result = pipeline.run(str(second), raw="other/dir")
|
|
assert result.raw_reused is True
|
|
assert result.raw_path == "raw/articles/one.txt"
|
|
assert result.duplicate_page == "wiki/sources/one.md"
|
|
copies = [path for path in (root / "raw").rglob("*") if path.is_file() and path.suffix == ".txt"]
|
|
assert copies == [root / "raw" / "articles" / "one.txt"]
|
|
assert "already archived at raw/articles/one.txt; --raw ignored" in result.warnings
|
|
|
|
|
|
def test_changed_file_gets_a_second_raw_name(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.3.4: a changed file of the same name is stored as ``slug-2``."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "notes.txt"
|
|
source.write_text("first\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(source), "--raw-only")
|
|
assert code == 0, stderr
|
|
original = (root / "raw" / "articles" / "notes.txt").read_bytes()
|
|
source.write_text("second\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
assert (root / "raw" / "articles" / "notes.txt").read_bytes() == original
|
|
assert (root / "raw" / "articles" / "notes-2.txt").read_text(encoding="utf-8") == "second\n"
|
|
|
|
|
|
def test_duplicate_exits_zero_and_skips_the_log(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.3.5: the same bytes report the first page and do not rewrite it."""
|
|
root = _wiki(tmp_path)
|
|
source = root / "ingest" / "a.txt"
|
|
source.write_text("alpha\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
page = root / "wiki" / "sources" / "a.md"
|
|
page_bytes = page.read_bytes()
|
|
log_before = (root / "wiki" / "log.md").read_text(encoding="utf-8")
|
|
again = root / "ingest" / "b.txt"
|
|
again.write_text("alpha\n", encoding="utf-8", newline="\n")
|
|
code, stdout, stderr = _run(root, str(again), "--json")
|
|
assert code == 0, stderr
|
|
payload = json.loads(stdout)
|
|
assert payload["duplicate_page"] == "wiki/sources/a.md"
|
|
assert payload["page_path"] is None
|
|
assert not again.exists()
|
|
assert page.read_bytes() == page_bytes
|
|
assert (root / "wiki" / "log.md").read_text(encoding="utf-8") == log_before
|
|
|
|
|
|
def test_ingest_drop_moves_and_personal_is_copied(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.3.6: ``ingest/`` is moved. A file outside it is copied."""
|
|
root = _wiki(tmp_path)
|
|
dropped = root / "ingest" / "a.pdf"
|
|
dropped.write_bytes(_pdf(["Moved"]))
|
|
code, _stdout, stderr = _run(root, str(dropped))
|
|
assert code == 0, stderr
|
|
assert not dropped.exists()
|
|
archived = root / "raw" / "papers" / "a.pdf"
|
|
assert archived.read_bytes() == _pdf(["Moved"])
|
|
personal = root / "personal" / "note.txt"
|
|
personal.write_text("keep me\n", encoding="utf-8", newline="\n")
|
|
original = personal.read_bytes()
|
|
code, _stdout, stderr = _run(root, str(personal))
|
|
assert code == 0, stderr
|
|
assert personal.read_bytes() == original
|
|
assert (root / "raw" / "articles" / "note.txt").read_bytes() == original
|
|
|
|
|
|
def test_origin_inside_wiki_is_relative(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.2.10: a file inside the wiki is recorded relative to the wiki root."""
|
|
root = _wiki(tmp_path)
|
|
dropped = root / "ingest" / "sub" / "a.txt"
|
|
dropped.parent.mkdir()
|
|
dropped.write_text("inside\n", encoding="utf-8", newline="\n")
|
|
code, stdout, stderr = _run(root, str(dropped.resolve()), "--json")
|
|
assert code == 0, stderr
|
|
assert json.loads(stdout)["origin"] == "ingest/sub/a.txt"
|
|
front = _front(root / "wiki" / "sources" / "a.md")
|
|
assert front["ingest"]["origin"] == "ingest/sub/a.txt"
|
|
assert front["ingest"]["origin_kind"] == "file"
|
|
assert "ingest ingest/sub/a.txt -> " in (root / "wiki" / "log.md").read_text(encoding="utf-8")
|
|
outside = tmp_path / "b.txt"
|
|
outside.write_text("outside\n", encoding="utf-8", newline="\n")
|
|
code, stdout, stderr = _run(root, str(outside), "--json")
|
|
assert code == 0, stderr
|
|
assert json.loads(stdout)["origin"] == str(outside.resolve())
|
|
|
|
|
|
def test_unknown_bytes_archive_without_a_page(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.4.1: random bytes exit 3 under ``raw/other``."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "blob.xyz"
|
|
source.write_bytes(b"\x00\x01\x02\xff")
|
|
code, stdout, _stderr = _run(root, str(source), "--json")
|
|
assert code == 3
|
|
payload = json.loads(stdout)
|
|
assert payload["conversion"] == "none"
|
|
assert payload["page_path"] is None
|
|
assert payload["raw_path"] == "raw/other/blob.xyz"
|
|
assert (root / "raw" / "other" / "blob.xyz").is_file()
|
|
assert not (root / "wiki" / "sources" / "blob.md").exists()
|
|
human_source = tmp_path / "other.xyz"
|
|
human_source.write_bytes(b"\x00\x02")
|
|
code, human, _stderr = _run(root, str(human_source))
|
|
assert code == 3
|
|
assert human.strip().splitlines()[-1].startswith("No converter for")
|
|
|
|
|
|
def test_encrypted_pdf_keeps_raw_and_skips_the_page(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.4.2: an encrypted PDF exits 3 and the raw copy stays."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "secret.pdf"
|
|
source.write_bytes(_pdf(["hidden"], password="secret"))
|
|
code, stdout, stderr = _run(root, str(source))
|
|
assert code == 3
|
|
assert "Conversion failed: pdf:" in stderr
|
|
assert (root / "raw" / "papers" / "secret.pdf").is_file()
|
|
assert not (root / "wiki" / "sources" / "secret.md").exists()
|
|
assert stdout.strip().splitlines()[-1].startswith("No converter for")
|
|
|
|
|
|
def test_html_keeps_article_link_and_table(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.4.5: article text, no script, an absolute link, and a pipe table."""
|
|
root = _wiki(tmp_path)
|
|
html = """<!doctype html>
|
|
<html lang="en"><head><title>Page Title</title></head>
|
|
<body>
|
|
<nav>NAVSECRET</nav>
|
|
<article>
|
|
<p>ARTICLETEX</p>
|
|
<a href="/a">Go</a>
|
|
<table><tr><th>Col</th></tr><tr><td>Cell</td></tr></table>
|
|
</article>
|
|
<script>SCRIPTSECRET</script>
|
|
</body></html>
|
|
"""
|
|
opener = _FakeOpener(html.encode("utf-8"), "text/html")
|
|
pipeline = default_pipeline(root)
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
|
|
result = pipeline.run("https://host/p")
|
|
assert result.exit_code == 0
|
|
body = (root / result.page_path).read_text(encoding="utf-8")
|
|
assert "ARTICLETEX" in body
|
|
assert "SCRIPTSECRET" not in body
|
|
assert "https://host/a" in body
|
|
assert "|" in body
|
|
|
|
|
|
def test_pdf_page_headings_and_blank_page(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.4.6: pages are numbered, and a blank page is partial."""
|
|
root = _wiki(tmp_path)
|
|
numbered = tmp_path / "two.pdf"
|
|
numbered.write_bytes(_pdf(["Alpha", "Beta"]))
|
|
code, _stdout, stderr = _run(root, str(numbered))
|
|
assert code == 0, stderr
|
|
body = (root / "wiki" / "sources" / "two.md").read_text(encoding="utf-8")
|
|
assert "## Page 1" in body
|
|
assert "## Page 2" in body
|
|
blank = tmp_path / "gap.pdf"
|
|
blank.write_bytes(_pdf(["Alpha", ""]))
|
|
code, _stdout, stderr = _run(root, str(blank))
|
|
assert code == 0, stderr
|
|
front = _front(root / "wiki" / "sources" / "gap.md")
|
|
assert front["ingest"]["conversion"] == "partial"
|
|
assert "page 2 has no text layer" in front["ingest"]["warnings"]
|
|
|
|
|
|
def test_target_paths(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.5.1: ``-t concepts`` writes under concepts. Escapes exit 1."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "idea.txt"
|
|
source.write_text("idea\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(source), "-t", "concepts")
|
|
assert code == 0, stderr
|
|
assert (root / "wiki" / "concepts" / "idea.md").is_file()
|
|
code, stdout, stderr = _run(root, str(source), "-t", "../raw/x.md")
|
|
assert code == 1
|
|
assert stdout == ""
|
|
assert "--target must stay inside wiki/" in stderr
|
|
other = tmp_path / "other.txt"
|
|
other.write_text("x\n", encoding="utf-8", newline="\n")
|
|
code, stdout, stderr = _run(root, str(other), "-t", "index.md")
|
|
assert code == 1
|
|
assert "--target cannot replace" in stderr
|
|
|
|
|
|
def test_existing_page_refuses_until_force(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.5.2: a second run exits 1. ``--force`` replaces the page."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "item.txt"
|
|
source.write_text("one\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
page = root / "wiki" / "sources" / "item.md"
|
|
before = page.read_bytes()
|
|
opener = _FakeOpener(b"nope", "text/plain")
|
|
pipeline = default_pipeline(root)
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=opener)
|
|
with pytest.raises(IngestError, match="Page exists:"):
|
|
pipeline.run("https://host/item")
|
|
assert opener.calls == 0
|
|
assert page.read_bytes() == before
|
|
source.write_text("two\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(source), "--force")
|
|
assert code == 0, stderr
|
|
assert page.read_bytes() != before
|
|
assert "two" in (root / "raw" / "articles" / "item-2.txt").read_text(encoding="utf-8")
|
|
|
|
|
|
def test_page_frontmatter_matches_raw_hash(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.5.3 and 7.1.5.5.4: frontmatter is complete and the page has one H1."""
|
|
root = _wiki(tmp_path)
|
|
index_before = (root / "wiki" / "index.md").read_bytes()
|
|
source = tmp_path / "story.md"
|
|
source.write_text("# Story\n\nSee [gone](missing.md).\n", encoding="utf-8", newline="\n")
|
|
pipeline = default_pipeline(root)
|
|
result = pipeline.run(str(source))
|
|
assert result.exit_code == 0
|
|
page = root / result.page_path
|
|
front = _front(page)
|
|
for key in _FRONTMATTER_KEYS:
|
|
assert key in front
|
|
for key in _INGEST_KEYS:
|
|
assert key in front["ingest"]
|
|
raw = root / front["sources"][0]
|
|
assert front["ingest"]["sha256"] == hashlib.sha256(raw.read_bytes()).hexdigest()
|
|
assert front["ingest"]["warnings"] == result.warnings
|
|
assert front["status"] == "raw"
|
|
assert front["confidence"] == "zero"
|
|
assert front["llm"] == [{"model": "none"}]
|
|
headings = [line for line in page.read_text(encoding="utf-8").splitlines() if line.startswith("# ")]
|
|
assert headings == ["# Story"]
|
|
linter = FrontmatterLinter(settings_file=root / "fiona-settings.json")
|
|
assert linter.lint_file(page) == []
|
|
links = MarkdownLinter(settings_file=root / "fiona-settings.json")
|
|
assert [item for item in links.lint_file(page) if item.code == "MDL001"] == []
|
|
assert (root / "wiki" / "index.md").read_bytes() == index_before
|
|
log = (root / "wiki" / "log.md").read_text(encoding="utf-8")
|
|
assert "version: 2.0.1" in log
|
|
assert f"ingest {source.resolve()} -> raw/articles/story.md -> wiki/sources/story.md (raw)" in log
|
|
|
|
|
|
def test_style_violations_become_one_warning(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.5.5: violations the fixer leaves are counted in one warning."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "long.md"
|
|
source.write_text("# Title\n\n" + ("word " * 40) + "\n", encoding="utf-8", newline="\n")
|
|
code, _stdout, stderr = _run(root, str(source))
|
|
assert code == 0, stderr
|
|
warnings = _front(root / "wiki" / "sources" / "long.md")["ingest"]["warnings"]
|
|
assert any(warning.endswith("markdown style violations left") for warning in warnings)
|
|
|
|
|
|
def test_raw_only_and_dry_run(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.7 and 7.1.5.8: raw-only skips the page. A dry run writes nothing."""
|
|
root = _wiki(tmp_path)
|
|
source = root / "ingest" / "only.txt"
|
|
source.write_text("archive me\n", encoding="utf-8", newline="\n")
|
|
code, stdout, stderr = _run(root, str(source), "--raw-only", "--json")
|
|
assert code == 0, stderr
|
|
payload = json.loads(stdout)
|
|
assert payload["conversion"] == "none"
|
|
assert payload["page_path"] is None
|
|
assert not source.exists()
|
|
assert (root / "raw" / "articles" / "only.txt").is_file()
|
|
kept = root / "ingest" / "stay.txt"
|
|
kept.write_text("stay\n", encoding="utf-8", newline="\n")
|
|
before = _hashes(root)
|
|
code, stdout, stderr = _run(root, str(kept), "--dry-run")
|
|
assert code == 0, stderr
|
|
assert stdout.startswith("Dry run - no files will be modified:")
|
|
assert kept.is_file()
|
|
assert _hashes(root) == before
|
|
code, stdout, _stderr = _run(root, str(kept), "--dry-run", "--json")
|
|
assert json.loads(stdout)["dry_run"] is True
|
|
|
|
|
|
def test_pending_hides_refined_pages(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.9: ``--pending`` lists raw pages and drops refined ones."""
|
|
root = _wiki(tmp_path)
|
|
source = tmp_path / "open.txt"
|
|
source.write_text("open\n", encoding="utf-8", newline="\n")
|
|
assert _run(root, str(source))[0] == 0
|
|
code, stdout, _stderr = _run(root, "--pending", "--json")
|
|
payload = json.loads(stdout)
|
|
assert code == 0
|
|
assert payload["total"] == 1
|
|
page = root / "wiki" / "sources" / "open.md"
|
|
text = page.read_text(encoding="utf-8").replace("status: raw", "status: refined", 1)
|
|
page.write_text(text, encoding="utf-8", newline="\n")
|
|
code, stdout, _stderr = _run(root, "--pending", "--json")
|
|
assert json.loads(stdout)["pending"] == []
|
|
|
|
|
|
def test_not_a_wiki_and_mutually_exclusive_args(tmp_path: pathlib.Path) -> None:
|
|
"""A directory without settings exits 1. Source and ``--pending`` cannot combine."""
|
|
code, stdout, stderr = _run(tmp_path, str(tmp_path / "a.txt"))
|
|
assert code == 1
|
|
assert stdout == ""
|
|
assert "is not an llm-wiki directory" in stderr
|
|
with pytest.raises(SystemExit) as caught:
|
|
ingest_main(["--pending", "file.txt"])
|
|
assert caught.value.code == 2
|
|
|
|
|
|
def test_sanity_ignores_raw_markdown(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.12: a raw markdown file with style violations does not fail sanity."""
|
|
root = _wiki(tmp_path)
|
|
raw = root / "raw" / "articles"
|
|
raw.mkdir(parents=True)
|
|
(raw / "bad-style.md").write_text(
|
|
"# Bad\n\n" + ("x" * 180) + "\n\n[gone](missing-file.md)\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
results = run_wiki_checks(root)
|
|
assert "bad-style.md" not in str(results)
|