Files
nvaldez de28441943 feat: sync Nokia fork product through SPEC 1.18 (v1.16.0)
MEMORY.md and skill memory template, ingest auth and html_sweep, TOC slug fix, run_tests CI output, and matching tests. Excludes GitLab CI and publishing docs.
2026-10-19 09:12:04 +02:00

351 lines
14 KiB
Python

"""Tests for login detection, exit 4, and ``--origin`` (SPEC 7.1.5.2.8, 7.1.5.2.9).
Generated by AI/LLM: 2026-10-07 - fake openers only; no test makes a network call.
"""
from __future__ import annotations
import contextlib
import email.message
import hashlib
import io
import json
import pathlib
import shutil
import typing
import urllib.error
import urllib.request
import pytest
import yaml
from fiona.paths import llm_wiki_template
from fiona.wiki.ingest import pipeline as pipeline_mod
from fiona.wiki.ingest.converters import default_registry
from fiona.wiki.ingest.fetchers import UrlFetcher
from fiona.wiki.ingest.models import AuthRequiredError, IngestError
from fiona.wiki.ingest.pipeline import default_pipeline, ingest_main
from fiona.wiki.ingest.settings import DEFAULT_LOGIN_URL_PATTERNS, load_ingest_settings
_NEXT = "Next: save it into ingest/, then run: wiki ingest <file> --origin "
class _Response:
"""One canned 200 response."""
def __init__(self, body: bytes, header: str, url: str) -> None:
self._body = body
self._offset = 0
self.headers = email.message.Message()
self.headers["Content-Type"] = header
self._url = url
def read(self, amount: int = -1) -> bytes:
"""Return the next chunk of the body."""
if amount < 0:
amount = len(self._body) - self._offset
chunk = self._body[self._offset:self._offset + amount]
self._offset += len(chunk)
return chunk
def geturl(self) -> str:
"""Return the URL after redirects."""
return self._url
def __enter__(self) -> typing.Self:
return self
def __exit__(self, *_args: object) -> bool:
return False
class _Opener:
"""Opener that answers one status, or 200 with a body at ``final_url``."""
def __init__(
self,
body: bytes = b"",
header: str = "text/html",
*,
status: int = 200,
final_url: str | None = None,
) -> None:
self.body = body
self.header = header
self.status = status
self.final_url = final_url
self.calls = 0
def open(self, request: urllib.request.Request, timeout: float | None = None) -> _Response:
"""Return the canned response or raise the canned HTTP error."""
del timeout
self.calls += 1
if self.status != 200:
raise urllib.error.HTTPError(
request.full_url,
self.status,
"server phrase",
email.message.Message(),
io.BytesIO(b""),
)
return _Response(self.body, self.header, self.final_url or request.full_url)
def _wiki(tmp_path: pathlib.Path, ingest: dict | None = None) -> pathlib.Path:
"""Create a small llm-wiki, optionally with ``fiona.wiki.ingest`` settings."""
root = tmp_path / "wiki"
(root / "wiki").mkdir(parents=True)
(root / "ingest").mkdir()
(root / "raw").mkdir()
shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json")
if ingest is not None:
settings = json.loads((root / "fiona-settings.json").read_text(encoding="utf-8"))
settings.setdefault("fiona", {}).setdefault("wiki", {})["ingest"] = ingest
(root / "fiona-settings.json").write_text(json.dumps(settings), encoding="utf-8")
(root / "wiki" / "log.md").write_text(
"---\nname: log\nversion: 2.0.0\nupdated: 2026-09-30\n---\n\n# Log\n",
encoding="utf-8",
newline="\n",
)
return root
def _hashes(root: pathlib.Path) -> dict[str, str]:
"""Hash every file under a wiki."""
return {
path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
for path in root.rglob("*")
if path.is_file()
}
def _cli(
root: pathlib.Path,
opener: _Opener | None,
monkeypatch: pytest.MonkeyPatch,
*args: str,
) -> tuple[int, str, str]:
"""Run the ingest CLI with the URL fetcher bound to ``opener``."""
if opener is not None:
original = pipeline_mod.default_pipeline
def patched(wiki_root: pathlib.Path) -> pipeline_mod.IngestPipeline:
built = original(wiki_root)
built.fetchers[0] = UrlFetcher(built.registry, opener=opener)
return built
monkeypatch.setattr(pipeline_mod, "default_pipeline", patched)
stdout = io.StringIO()
stderr = io.StringIO()
with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
code = ingest_main(["--wiki", str(root), *args])
return code, stdout.getvalue(), stderr.getvalue()
def _front(path: pathlib.Path) -> dict:
"""Parse page frontmatter."""
return yaml.safe_load(path.read_text(encoding="utf-8").split("---", 2)[1])
@pytest.mark.parametrize("status, phrase", [(401, "401 Unauthorized"), (403, "403 Forbidden")])
def test_status_401_403_exit_4_and_write_nothing(
tmp_path: pathlib.Path,
monkeypatch: pytest.MonkeyPatch,
status: int,
phrase: str,
) -> None:
"""SPEC 7.1.5.2.8: 401 and 403 exit 4 with both stderr lines and no file change."""
root = _wiki(tmp_path)
before = _hashes(root)
code, stdout, stderr = _cli(root, _Opener(status=status), monkeypatch, "https://host/job/x/console")
assert code == 4
assert stdout == ""
lines = stderr.strip().splitlines()
assert lines == [
f"Error: Authentication required: https://host/job/x/console: {phrase}",
_NEXT + "https://host/job/x/console",
]
assert "Traceback" not in stderr
assert _hashes(root) == before
def test_redirect_to_login_exits_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""SPEC 7.1.5.2.8.2: a redirect whose host and path match a pattern is a login."""
root = _wiki(tmp_path)
before = _hashes(root)
final = "https://host/login?from=%2Fjob%2Fx%2Fconsole"
opener = _Opener(b"<html><body><p>Sign in to continue please</p></body></html>", final_url=final)
code, _stdout, stderr = _cli(root, opener, monkeypatch, "https://host/job/x/console")
assert code == 4
assert f"redirected to {final}" in stderr
assert _hashes(root) == before
def test_login_form_exits_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""SPEC 7.1.5.2.8.2: an HTML body with a password input is a login page."""
root = _wiki(tmp_path)
before = _hashes(root)
body = b"<html><body><form><INPUT TYPE='password' name=p></form></body></html>"
code, _stdout, stderr = _cli(root, _Opener(body), monkeypatch, "https://host/wiki/page")
assert code == 4
assert "login page at https://host/wiki/page" in stderr
assert _hashes(root) == before
def test_login_form_unquoted_type() -> None:
"""The password check accepts unquoted and double-quoted ``type`` values."""
registry = default_registry()
for markup in (b'<input type="password">', b"<input name=a type=PASSWORD>"):
fetcher = UrlFetcher(registry, opener=_Opener(b"<html><body>" + markup + b"</body></html>"))
with pytest.raises(AuthRequiredError, match="login page at"):
fetcher.fetch("https://host/p", pathlib.Path("unused"))
def test_docs_login_page_without_redirect_ingests(tmp_path: pathlib.Path) -> None:
"""SPEC 7.1.5.2.8.3: a page whose own path holds ``login`` still ingests."""
root = _wiki(tmp_path)
body = b"<html><body><p>How to configure the login screen for users</p></body></html>"
pipeline = default_pipeline(root)
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body))
result = pipeline.run("https://host/docs/login")
assert result.exit_code == 0
assert result.page_path is not None
def test_redirect_elsewhere_ingests(tmp_path: pathlib.Path) -> None:
"""A redirect to a path that matches no pattern is a normal download."""
root = _wiki(tmp_path)
body = b"<html><body><p>Moved content that is still the real page</p></body></html>"
pipeline = default_pipeline(root)
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://host/new"))
assert pipeline.run("https://host/old").exit_code == 0
def test_404_still_download_failed(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""SPEC 7.1.5.2.3: other non-2xx statuses keep exit 1."""
root = _wiki(tmp_path)
code, _stdout, stderr = _cli(root, _Opener(status=404), monkeypatch, "https://host/missing")
assert code == 1
assert "Download failed" in stderr
assert "Next:" not in stderr
def test_json_and_dry_run_exit_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""SPEC 7.1.5.2.8.5: ``--json`` prints nothing on stdout; ``--dry-run`` exits 4."""
root = _wiki(tmp_path)
before = _hashes(root)
opener = _Opener(status=401)
code, stdout, stderr = _cli(root, opener, monkeypatch, "https://host/a", "--json")
assert (code, stdout) == (4, "")
assert "Authentication required" in stderr
code, stdout, stderr = _cli(root, None, monkeypatch, "https://host/a", "--dry-run")
assert (code, stdout) == (4, "")
assert _hashes(root) == before
def test_userinfo_removed_from_both_lines(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""SPEC 7.1.5.2.8.4: the URL in both lines has no userinfo."""
root = _wiki(tmp_path)
code, _stdout, stderr = _cli(root, _Opener(status=401), monkeypatch, "https://bob:pw@host/a")
assert code == 4
assert "pw" not in stderr
assert "bob" not in stderr
assert "https://host/a" in stderr
def test_auth_error_carries_url_and_reason() -> None:
"""``AuthRequiredError`` is an ``IngestError`` with exit 4."""
error = AuthRequiredError("https://host/a", "401 Unauthorized")
assert isinstance(error, IngestError)
assert error.exit_code == 4
assert error.url == "https://host/a"
assert error.reason == "401 Unauthorized"
assert str(error) == "Authentication required: https://host/a: 401 Unauthorized"
def test_login_url_patterns_settings(tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture) -> None:
"""SPEC 5.1: default list, user override, and one bad regex skipped with a warning."""
assert load_ingest_settings(tmp_path).login_url_patterns == list(DEFAULT_LOGIN_URL_PATTERNS)
root = _wiki(tmp_path, {"login_url_patterns": ["/auth/", "(", 7]})
with caplog.at_level("WARNING"):
loaded = load_ingest_settings(root)
assert loaded.login_url_patterns == ["/auth/"]
assert sum("login_url_patterns" in record.getMessage() for record in caplog.records) == 2
other = _wiki(tmp_path / "other", {"login_url_patterns": "not a list"})
assert load_ingest_settings(other).login_url_patterns == list(DEFAULT_LOGIN_URL_PATTERNS)
def test_user_pattern_replaces_default(tmp_path: pathlib.Path) -> None:
"""A user list replaces the default, so ``/login`` is no longer a hit."""
root = _wiki(tmp_path, {"login_url_patterns": ["/gate/"]})
pipeline = default_pipeline(root)
body = b"<html><body><p>Some ordinary page text for the test</p></body></html>"
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://host/login"))
assert pipeline.run("https://host/a").exit_code == 0
pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://HOST/Gate/x"))
with pytest.raises(AuthRequiredError, match="redirected to"):
pipeline.run("https://host/b")
def test_origin_sets_provenance_and_base_url(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""SPEC 7.1.5.2.9: a saved file keeps the URL it came from, with no network call."""
root = _wiki(tmp_path)
saved = root / "ingest" / "console.html"
saved.write_text(
'<html><body><p>Build output that was saved by the user <a href="/a">here</a></p></body></html>',
encoding="utf-8",
)
opener = _Opener(status=500)
code, stdout, stderr = _cli(
root,
opener,
monkeypatch,
str(saved),
"--origin",
"https://bob:pw@host/job/x/console",
"--json",
)
assert code == 0, stderr
assert opener.calls == 0
payload = json.loads(stdout)
assert payload["origin"] == "https://host/job/x/console"
assert payload["origin_kind"] == "url"
page = root / payload["page_path"]
assert page.name == "console.md"
front = _front(page)
assert front["ingest"]["origin"] == "https://host/job/x/console"
assert front["ingest"]["origin_kind"] == "url"
assert front["ingest"]["final_url"] is None
assert "https://host/a" in page.read_text(encoding="utf-8")
log = (root / "wiki" / "log.md").read_text(encoding="utf-8")
assert "ingest https://host/job/x/console -> raw/articles/console.html" in log
def test_origin_human_output_has_no_fetched_line(tmp_path: pathlib.Path) -> None:
"""A local file with ``--origin`` was not downloaded, so there is no ``Fetched:`` line."""
root = _wiki(tmp_path)
saved = tmp_path / "notes.txt"
saved.write_text("saved text\n", encoding="utf-8")
result = default_pipeline(root).run(str(saved), origin="https://host/notes")
assert not any(line.startswith("Fetched:") for line in result.messages)
def test_origin_rejects_url_source_and_bad_scheme(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None:
"""SPEC 7.1.5.2.9.1: both checks exit 1 before any fetch."""
root = _wiki(tmp_path)
before = _hashes(root)
opener = _Opener(b"x")
code, _stdout, stderr = _cli(root, opener, monkeypatch, "https://host/a", "--origin", "https://host/b")
assert code == 1
assert "Error: --origin needs a local file: https://host/a" in stderr
saved = tmp_path / "a.txt"
saved.write_text("a\n", encoding="utf-8")
code, _stdout, stderr = _cli(root, None, monkeypatch, str(saved), "--origin", "ftp://host/a")
assert code == 1
assert "Error: --origin must be an http or https URL: ftp://host/a" in stderr
code, _stdout, stderr = _cli(root, None, monkeypatch, str(saved), "--origin", "host/a")
assert code == 1
assert "--origin must be an http or https URL" in stderr
assert opener.calls == 0
assert _hashes(root) == before