de28441943
MEMORY.md and skill memory template, ingest auth and html_sweep, TOC slug fix, run_tests CI output, and matching tests. Excludes GitLab CI and publishing docs.
351 lines
14 KiB
Python
351 lines
14 KiB
Python
"""Tests for login detection, exit 4, and ``--origin`` (SPEC 7.1.5.2.8, 7.1.5.2.9).
|
|
|
|
Generated by AI/LLM: 2026-10-07 - fake openers only; no test makes a network call.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import email.message
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import pathlib
|
|
import shutil
|
|
import typing
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
from fiona.paths import llm_wiki_template
|
|
from fiona.wiki.ingest import pipeline as pipeline_mod
|
|
from fiona.wiki.ingest.converters import default_registry
|
|
from fiona.wiki.ingest.fetchers import UrlFetcher
|
|
from fiona.wiki.ingest.models import AuthRequiredError, IngestError
|
|
from fiona.wiki.ingest.pipeline import default_pipeline, ingest_main
|
|
from fiona.wiki.ingest.settings import DEFAULT_LOGIN_URL_PATTERNS, load_ingest_settings
|
|
|
|
_NEXT = "Next: save it into ingest/, then run: wiki ingest <file> --origin "
|
|
|
|
|
|
class _Response:
|
|
"""One canned 200 response."""
|
|
|
|
def __init__(self, body: bytes, header: str, url: str) -> None:
|
|
self._body = body
|
|
self._offset = 0
|
|
self.headers = email.message.Message()
|
|
self.headers["Content-Type"] = header
|
|
self._url = url
|
|
|
|
def read(self, amount: int = -1) -> bytes:
|
|
"""Return the next chunk of the body."""
|
|
if amount < 0:
|
|
amount = len(self._body) - self._offset
|
|
chunk = self._body[self._offset:self._offset + amount]
|
|
self._offset += len(chunk)
|
|
return chunk
|
|
|
|
def geturl(self) -> str:
|
|
"""Return the URL after redirects."""
|
|
return self._url
|
|
|
|
def __enter__(self) -> typing.Self:
|
|
return self
|
|
|
|
def __exit__(self, *_args: object) -> bool:
|
|
return False
|
|
|
|
|
|
class _Opener:
|
|
"""Opener that answers one status, or 200 with a body at ``final_url``."""
|
|
|
|
def __init__(
|
|
self,
|
|
body: bytes = b"",
|
|
header: str = "text/html",
|
|
*,
|
|
status: int = 200,
|
|
final_url: str | None = None,
|
|
) -> None:
|
|
self.body = body
|
|
self.header = header
|
|
self.status = status
|
|
self.final_url = final_url
|
|
self.calls = 0
|
|
|
|
def open(self, request: urllib.request.Request, timeout: float | None = None) -> _Response:
|
|
"""Return the canned response or raise the canned HTTP error."""
|
|
del timeout
|
|
self.calls += 1
|
|
if self.status != 200:
|
|
raise urllib.error.HTTPError(
|
|
request.full_url,
|
|
self.status,
|
|
"server phrase",
|
|
email.message.Message(),
|
|
io.BytesIO(b""),
|
|
)
|
|
return _Response(self.body, self.header, self.final_url or request.full_url)
|
|
|
|
|
|
def _wiki(tmp_path: pathlib.Path, ingest: dict | None = None) -> pathlib.Path:
|
|
"""Create a small llm-wiki, optionally with ``fiona.wiki.ingest`` settings."""
|
|
root = tmp_path / "wiki"
|
|
(root / "wiki").mkdir(parents=True)
|
|
(root / "ingest").mkdir()
|
|
(root / "raw").mkdir()
|
|
shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json")
|
|
if ingest is not None:
|
|
settings = json.loads((root / "fiona-settings.json").read_text(encoding="utf-8"))
|
|
settings.setdefault("fiona", {}).setdefault("wiki", {})["ingest"] = ingest
|
|
(root / "fiona-settings.json").write_text(json.dumps(settings), encoding="utf-8")
|
|
(root / "wiki" / "log.md").write_text(
|
|
"---\nname: log\nversion: 2.0.0\nupdated: 2026-09-30\n---\n\n# Log\n",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
)
|
|
return root
|
|
|
|
|
|
def _hashes(root: pathlib.Path) -> dict[str, str]:
|
|
"""Hash every file under a wiki."""
|
|
return {
|
|
path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
|
|
for path in root.rglob("*")
|
|
if path.is_file()
|
|
}
|
|
|
|
|
|
def _cli(
|
|
root: pathlib.Path,
|
|
opener: _Opener | None,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
*args: str,
|
|
) -> tuple[int, str, str]:
|
|
"""Run the ingest CLI with the URL fetcher bound to ``opener``."""
|
|
if opener is not None:
|
|
original = pipeline_mod.default_pipeline
|
|
|
|
def patched(wiki_root: pathlib.Path) -> pipeline_mod.IngestPipeline:
|
|
built = original(wiki_root)
|
|
built.fetchers[0] = UrlFetcher(built.registry, opener=opener)
|
|
return built
|
|
|
|
monkeypatch.setattr(pipeline_mod, "default_pipeline", patched)
|
|
stdout = io.StringIO()
|
|
stderr = io.StringIO()
|
|
with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
|
|
code = ingest_main(["--wiki", str(root), *args])
|
|
return code, stdout.getvalue(), stderr.getvalue()
|
|
|
|
|
|
def _front(path: pathlib.Path) -> dict:
|
|
"""Parse page frontmatter."""
|
|
return yaml.safe_load(path.read_text(encoding="utf-8").split("---", 2)[1])
|
|
|
|
|
|
@pytest.mark.parametrize("status, phrase", [(401, "401 Unauthorized"), (403, "403 Forbidden")])
|
|
def test_status_401_403_exit_4_and_write_nothing(
|
|
tmp_path: pathlib.Path,
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
status: int,
|
|
phrase: str,
|
|
) -> None:
|
|
"""SPEC 7.1.5.2.8: 401 and 403 exit 4 with both stderr lines and no file change."""
|
|
root = _wiki(tmp_path)
|
|
before = _hashes(root)
|
|
code, stdout, stderr = _cli(root, _Opener(status=status), monkeypatch, "https://host/job/x/console")
|
|
assert code == 4
|
|
assert stdout == ""
|
|
lines = stderr.strip().splitlines()
|
|
assert lines == [
|
|
f"Error: Authentication required: https://host/job/x/console: {phrase}",
|
|
_NEXT + "https://host/job/x/console",
|
|
]
|
|
assert "Traceback" not in stderr
|
|
assert _hashes(root) == before
|
|
|
|
|
|
def test_redirect_to_login_exits_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""SPEC 7.1.5.2.8.2: a redirect whose host and path match a pattern is a login."""
|
|
root = _wiki(tmp_path)
|
|
before = _hashes(root)
|
|
final = "https://host/login?from=%2Fjob%2Fx%2Fconsole"
|
|
opener = _Opener(b"<html><body><p>Sign in to continue please</p></body></html>", final_url=final)
|
|
code, _stdout, stderr = _cli(root, opener, monkeypatch, "https://host/job/x/console")
|
|
assert code == 4
|
|
assert f"redirected to {final}" in stderr
|
|
assert _hashes(root) == before
|
|
|
|
|
|
def test_login_form_exits_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""SPEC 7.1.5.2.8.2: an HTML body with a password input is a login page."""
|
|
root = _wiki(tmp_path)
|
|
before = _hashes(root)
|
|
body = b"<html><body><form><INPUT TYPE='password' name=p></form></body></html>"
|
|
code, _stdout, stderr = _cli(root, _Opener(body), monkeypatch, "https://host/wiki/page")
|
|
assert code == 4
|
|
assert "login page at https://host/wiki/page" in stderr
|
|
assert _hashes(root) == before
|
|
|
|
|
|
def test_login_form_unquoted_type() -> None:
|
|
"""The password check accepts unquoted and double-quoted ``type`` values."""
|
|
registry = default_registry()
|
|
for markup in (b'<input type="password">', b"<input name=a type=PASSWORD>"):
|
|
fetcher = UrlFetcher(registry, opener=_Opener(b"<html><body>" + markup + b"</body></html>"))
|
|
with pytest.raises(AuthRequiredError, match="login page at"):
|
|
fetcher.fetch("https://host/p", pathlib.Path("unused"))
|
|
|
|
|
|
def test_docs_login_page_without_redirect_ingests(tmp_path: pathlib.Path) -> None:
|
|
"""SPEC 7.1.5.2.8.3: a page whose own path holds ``login`` still ingests."""
|
|
root = _wiki(tmp_path)
|
|
body = b"<html><body><p>How to configure the login screen for users</p></body></html>"
|
|
pipeline = default_pipeline(root)
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body))
|
|
result = pipeline.run("https://host/docs/login")
|
|
assert result.exit_code == 0
|
|
assert result.page_path is not None
|
|
|
|
|
|
def test_redirect_elsewhere_ingests(tmp_path: pathlib.Path) -> None:
|
|
"""A redirect to a path that matches no pattern is a normal download."""
|
|
root = _wiki(tmp_path)
|
|
body = b"<html><body><p>Moved content that is still the real page</p></body></html>"
|
|
pipeline = default_pipeline(root)
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://host/new"))
|
|
assert pipeline.run("https://host/old").exit_code == 0
|
|
|
|
|
|
def test_404_still_download_failed(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""SPEC 7.1.5.2.3: other non-2xx statuses keep exit 1."""
|
|
root = _wiki(tmp_path)
|
|
code, _stdout, stderr = _cli(root, _Opener(status=404), monkeypatch, "https://host/missing")
|
|
assert code == 1
|
|
assert "Download failed" in stderr
|
|
assert "Next:" not in stderr
|
|
|
|
|
|
def test_json_and_dry_run_exit_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""SPEC 7.1.5.2.8.5: ``--json`` prints nothing on stdout; ``--dry-run`` exits 4."""
|
|
root = _wiki(tmp_path)
|
|
before = _hashes(root)
|
|
opener = _Opener(status=401)
|
|
code, stdout, stderr = _cli(root, opener, monkeypatch, "https://host/a", "--json")
|
|
assert (code, stdout) == (4, "")
|
|
assert "Authentication required" in stderr
|
|
code, stdout, stderr = _cli(root, None, monkeypatch, "https://host/a", "--dry-run")
|
|
assert (code, stdout) == (4, "")
|
|
assert _hashes(root) == before
|
|
|
|
|
|
def test_userinfo_removed_from_both_lines(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""SPEC 7.1.5.2.8.4: the URL in both lines has no userinfo."""
|
|
root = _wiki(tmp_path)
|
|
code, _stdout, stderr = _cli(root, _Opener(status=401), monkeypatch, "https://bob:pw@host/a")
|
|
assert code == 4
|
|
assert "pw" not in stderr
|
|
assert "bob" not in stderr
|
|
assert "https://host/a" in stderr
|
|
|
|
|
|
def test_auth_error_carries_url_and_reason() -> None:
|
|
"""``AuthRequiredError`` is an ``IngestError`` with exit 4."""
|
|
error = AuthRequiredError("https://host/a", "401 Unauthorized")
|
|
assert isinstance(error, IngestError)
|
|
assert error.exit_code == 4
|
|
assert error.url == "https://host/a"
|
|
assert error.reason == "401 Unauthorized"
|
|
assert str(error) == "Authentication required: https://host/a: 401 Unauthorized"
|
|
|
|
|
|
def test_login_url_patterns_settings(tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture) -> None:
|
|
"""SPEC 5.1: default list, user override, and one bad regex skipped with a warning."""
|
|
assert load_ingest_settings(tmp_path).login_url_patterns == list(DEFAULT_LOGIN_URL_PATTERNS)
|
|
root = _wiki(tmp_path, {"login_url_patterns": ["/auth/", "(", 7]})
|
|
with caplog.at_level("WARNING"):
|
|
loaded = load_ingest_settings(root)
|
|
assert loaded.login_url_patterns == ["/auth/"]
|
|
assert sum("login_url_patterns" in record.getMessage() for record in caplog.records) == 2
|
|
other = _wiki(tmp_path / "other", {"login_url_patterns": "not a list"})
|
|
assert load_ingest_settings(other).login_url_patterns == list(DEFAULT_LOGIN_URL_PATTERNS)
|
|
|
|
|
|
def test_user_pattern_replaces_default(tmp_path: pathlib.Path) -> None:
|
|
"""A user list replaces the default, so ``/login`` is no longer a hit."""
|
|
root = _wiki(tmp_path, {"login_url_patterns": ["/gate/"]})
|
|
pipeline = default_pipeline(root)
|
|
body = b"<html><body><p>Some ordinary page text for the test</p></body></html>"
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://host/login"))
|
|
assert pipeline.run("https://host/a").exit_code == 0
|
|
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://HOST/Gate/x"))
|
|
with pytest.raises(AuthRequiredError, match="redirected to"):
|
|
pipeline.run("https://host/b")
|
|
|
|
|
|
def test_origin_sets_provenance_and_base_url(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""SPEC 7.1.5.2.9: a saved file keeps the URL it came from, with no network call."""
|
|
root = _wiki(tmp_path)
|
|
saved = root / "ingest" / "console.html"
|
|
saved.write_text(
|
|
'<html><body><p>Build output that was saved by the user <a href="/a">here</a></p></body></html>',
|
|
encoding="utf-8",
|
|
)
|
|
opener = _Opener(status=500)
|
|
code, stdout, stderr = _cli(
|
|
root,
|
|
opener,
|
|
monkeypatch,
|
|
str(saved),
|
|
"--origin",
|
|
"https://bob:pw@host/job/x/console",
|
|
"--json",
|
|
)
|
|
assert code == 0, stderr
|
|
assert opener.calls == 0
|
|
payload = json.loads(stdout)
|
|
assert payload["origin"] == "https://host/job/x/console"
|
|
assert payload["origin_kind"] == "url"
|
|
page = root / payload["page_path"]
|
|
assert page.name == "console.md"
|
|
front = _front(page)
|
|
assert front["ingest"]["origin"] == "https://host/job/x/console"
|
|
assert front["ingest"]["origin_kind"] == "url"
|
|
assert front["ingest"]["final_url"] is None
|
|
assert "https://host/a" in page.read_text(encoding="utf-8")
|
|
log = (root / "wiki" / "log.md").read_text(encoding="utf-8")
|
|
assert "ingest https://host/job/x/console -> raw/articles/console.html" in log
|
|
|
|
|
|
def test_origin_human_output_has_no_fetched_line(tmp_path: pathlib.Path) -> None:
|
|
"""A local file with ``--origin`` was not downloaded, so there is no ``Fetched:`` line."""
|
|
root = _wiki(tmp_path)
|
|
saved = tmp_path / "notes.txt"
|
|
saved.write_text("saved text\n", encoding="utf-8")
|
|
result = default_pipeline(root).run(str(saved), origin="https://host/notes")
|
|
assert not any(line.startswith("Fetched:") for line in result.messages)
|
|
|
|
|
|
def test_origin_rejects_url_source_and_bad_scheme(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""SPEC 7.1.5.2.9.1: both checks exit 1 before any fetch."""
|
|
root = _wiki(tmp_path)
|
|
before = _hashes(root)
|
|
opener = _Opener(b"x")
|
|
code, _stdout, stderr = _cli(root, opener, monkeypatch, "https://host/a", "--origin", "https://host/b")
|
|
assert code == 1
|
|
assert "Error: --origin needs a local file: https://host/a" in stderr
|
|
saved = tmp_path / "a.txt"
|
|
saved.write_text("a\n", encoding="utf-8")
|
|
code, _stdout, stderr = _cli(root, None, monkeypatch, str(saved), "--origin", "ftp://host/a")
|
|
assert code == 1
|
|
assert "Error: --origin must be an http or https URL: ftp://host/a" in stderr
|
|
code, _stdout, stderr = _cli(root, None, monkeypatch, str(saved), "--origin", "host/a")
|
|
assert code == 1
|
|
assert "--origin must be an http or https URL" in stderr
|
|
assert opener.calls == 0
|
|
assert _hashes(root) == before
|