Files
fiona-assistant/tests/test_html_sweep.py
T
nvaldez de28441943 feat: sync Nokia fork product through SPEC 1.18 (v1.16.0)
MEMORY.md and skill memory template, ingest auth and html_sweep, TOC slug fix, run_tests CI output, and matching tests. Excludes GitLab CI and publishing docs.
2026-10-19 09:12:04 +02:00

400 lines
18 KiB
Python

"""Tests for the HTML block sweep (SPEC 7.1.5.4.5.6) and its settings (SPEC 5.1).
Generated by AI/LLM: 2026-10-07 - one test per acceptance of 7.1.5.4.5.6.
"""
from __future__ import annotations
import json
import pathlib
import re
import shutil
import pytest
from fiona.paths import llm_wiki_template
from fiona.wiki.ingest.converters import default_registry
from fiona.wiki.ingest.converters.html import HtmlConverter
from fiona.wiki.ingest.models import ConversionResult, FetchedSource
from fiona.wiki.ingest.pipeline import default_pipeline
from fiona.wiki.ingest.settings import default_settings, load_ingest_settings
from fiona.wiki.sanity_manager import run_wiki_checks
_REVIEW = re.compile(r"<!-- fiona:review (\w+) -->\n\n(.*?)\n\n<!-- /fiona:review -->", re.DOTALL)
_LANDING = """<!doctype html>
<html><head><title>Agent harness</title></head><body>
<div class="hero"><section>
<h2>How it works</h2>
<div>Run every agent against the same suite of tests and compare results</div>
</section></div>
<article><blockquote>The harness cut our regression time in half overnight.</blockquote>
<p>Ada, Example Corp</p></article>
<article><blockquote>We ship agents with far more confidence than before.</blockquote>
<p>Bob, Other Inc</p></article>
</body></html>
"""
_BOILERPLATE = """<html><body>
<nav><ul><li><a href="/">Home</a></li><li><a href="/docs">Docs</a></li></ul></nav>
<main>
<p>Twelve words of real content sit in this one paragraph right here.</p>
<span>\u2022</span>
<div><span>Share</span></div>
<div><span>Share</span></div>
<p>Twelve words of real content sit in this one paragraph right here.</p>
<ul><li><a href="/x">Alpha link</a></li><li><a href="/y">Beta link</a></li></ul>
<div>Tiny note</div>
</main>
<footer><p>Copyright notice for this example site goes here.</p></footer>
</body></html>
"""
_ENCYCLOPEDIA = """<html><body>
<nav id="side"><ul><li><a href="/main">Main page</a></li><li><a href="/random">Random article</a></li></ul></nav>
<div id="content">
<h1>Ada Lovelace</h1>
<table class="infobox"><tr><th>Born</th><td>10 December 1815</td></tr>
<tr><th>Known for</th><td>Notes on the Analytical Engine</td></tr></table>
<p>Ada Lovelace was an English mathematician and writer.<sup><a href="#c1">[1]</a></sup></p>
<p>She is known for her work on the Analytical Engine of Charles Babbage.<sup><a href="#c2">[2]</a></sup></p>
<h2>References</h2>
<ol><li><a href="https://a.example/1">Toole, Ada: The Enchantress of Numbers</a></li>
<li><a href="https://a.example/2">Essinger, Ada's Algorithm</a></li></ol>
<table class="navbox"><tr><th><a href="/m">Mathematicians</a></th>
<td><a href="/b">Babbage</a> <a href="/c">Boole</a> <a href="/d">De Morgan</a></td></tr></table>
</div></body></html>
"""
_QA = """<html><body>
<div class="question"><div class="votes">42</div>
<div class="post"><p>How do I print every item of a list in reverse order?</p></div></div>
<div class="answer"><div class="votes">7</div>
<div class="post"><p>Loop over the reversed view so nothing is copied.</p>
<pre><code>for item in reversed(items): print(item)</code></pre></div></div>
<div class="answer"><div class="votes">3</div>
<div class="post"><p>The same loop also works on any sequence type.</p>
<pre><code>for item in reversed(items): print(item)</code></pre></div></div>
<aside><h3>Related questions</h3><ul><li><a href="/q/1">Sort a list</a></li></ul></aside>
</body></html>
"""
_FORUM = """<html><body><main><forum-thread>
<forum-post>First post body text written by the original author
<forum-reply>Reply number one adds a useful detail</forum-reply>
<forum-reply style="Display : None">HIDDENREPLY should never appear in output</forum-reply>
</forum-post>
<forum-post><template shadowrootmode="open"><p>Shadow post text that lives in a template</p></template></forum-post>
<forum-post>Third post closes the thread with thanks</forum-post>
</forum-thread></main></body></html>
"""
_LAYOUT_TABLE = """<html><body><table>
<tr><td><div>Post one has enough words to keep</div></td></tr>
<tr><td><div>Post two also has enough words here</div></td></tr>
</table></body></html>
"""
def _convert(tmp_path: pathlib.Path, html: str, **options: float) -> ConversionResult:
"""Convert one HTML string with the given sweep options."""
path = tmp_path / "page.html"
path.write_text(html, encoding="utf-8")
source = FetchedSource(
argument=str(path),
path=path,
origin=str(path),
origin_kind="file",
final_url=None,
media_type="text/html",
charset=None,
size_bytes=path.stat().st_size,
slug="page",
fetched="2026-10-07T00:00:00Z",
created=False,
)
return HtmlConverter(**options).convert(source)
def _reviews(body: str) -> list[tuple[str, str]]:
"""Return ``(reason, markdown)`` for every review block."""
return _REVIEW.findall(body)
def _outside(body: str) -> str:
"""Return the body with every review block removed."""
return _REVIEW.sub("", body)
def _wiki(tmp_path: pathlib.Path, html_settings: dict | None = None) -> pathlib.Path:
"""Create a small llm-wiki, optionally with ``fiona.wiki.ingest.html``."""
root = tmp_path / "wiki"
(root / "wiki").mkdir(parents=True)
(root / "ingest").mkdir()
(root / "raw").mkdir()
shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json")
if html_settings is not None:
settings = json.loads((root / "fiona-settings.json").read_text(encoding="utf-8"))
settings.setdefault("fiona", {}).setdefault("wiki", {})["ingest"] = {"html": html_settings}
(root / "fiona-settings.json").write_text(json.dumps(settings), encoding="utf-8")
(root / "wiki" / "log.md").write_text(
"---\nname: log\nversion: 2.0.0\nupdated: 2026-09-30\n---\n\n# Log\n",
encoding="utf-8",
newline="\n",
)
return root
def test_landing_page_keeps_every_section(tmp_path: pathlib.Path) -> None:
"""A section's heading and div, then both cards, all in order."""
body = _convert(tmp_path, _LANDING).body
outside = _outside(body)
order = [
"## How it works",
"Run every agent against the same suite of tests and compare results",
"The harness cut our regression time in half overnight.",
"We ship agents with far more confidence than before.",
]
positions = [outside.index(text) for text in order]
assert positions == sorted(positions)
def test_boilerplate_verdicts_and_reasons(tmp_path: pathlib.Path) -> None:
"""Nav and the bullet are dropped; chrome, repeat, links, and short are marked."""
result = _convert(tmp_path, _BOILERPLATE)
body = result.body
assert "Home" not in body
assert "Docs" not in body
assert "\u2022" not in body
assert body.count("Share") == 1
paragraph = "Twelve words of real content sit in this one paragraph right here."
assert paragraph in _outside(body)
reasons = dict((text.strip(), reason) for reason, text in _reviews(body))
assert reasons["Copyright notice for this example site goes here."] == "chrome"
assert reasons[paragraph] == "repeat"
assert reasons["Tiny note"] == "short"
assert reasons["Share"] == "short"
links = [text for text, reason in reasons.items() if reason == "links"]
assert len(links) == 1
assert "Alpha link" in links[0]
assert "html sweep: 1 keep, 5 review, 3 drop" in result.warnings
assert result.conversion == "complete"
def test_encyclopedia_page(tmp_path: pathlib.Path) -> None:
"""Body paragraphs keep; references and navbox are links; no menu text."""
body = _convert(tmp_path, _ENCYCLOPEDIA).body
outside = _outside(body)
assert "Ada Lovelace was an English mathematician and writer." in outside
assert "She is known for her work on the Analytical Engine of Charles Babbage." in outside
assert "Main page" not in body
assert "Random article" not in body
reasons = [(reason, text) for reason, text in _reviews(body)]
assert any(reason == "links" and "Enchantress" in text for reason, text in reasons)
assert any(reason == "links" and "Boole" in text for reason, text in reasons)
def test_qa_page(tmp_path: pathlib.Path) -> None:
"""Question and answers keep in order; repeated pre, vote counts, and aside are marked."""
body = _convert(tmp_path, _QA).body
outside = _outside(body)
order = [
"How do I print every item of a list in reverse order?",
"Loop over the reversed view so nothing is copied.",
"The same loop also works on any sequence type.",
]
positions = [outside.index(text) for text in order]
assert positions == sorted(positions)
assert outside.count("for item in reversed(items)") == 1
reasons = _reviews(body)
assert [reason for reason, text in reasons if "reversed(items)" in text] == ["repeat"]
assert sorted(text.strip() for reason, text in reasons if reason == "short") == ["3", "42", "7"]
assert {reason for reason, text in reasons if "Related questions" in text or "Sort a list" in text} == {
"chrome"
}
def test_forum_thread(tmp_path: pathlib.Path) -> None:
"""Custom-element posts, a shadow-root post, and replies in order; hidden reply gone."""
body = _convert(tmp_path, _FORUM).body
order = [
"First post body text written by the original author",
"Reply number one adds a useful detail",
"Shadow post text that lives in a template",
"Third post closes the thread with thanks",
]
positions = [_outside(body).index(text) for text in order]
assert positions == sorted(positions)
assert "HIDDENREPLY" not in body
def test_layout_table_gives_one_block_per_post(tmp_path: pathlib.Path) -> None:
"""Cells that hold divs are containers, not a pipe table."""
body = _convert(tmp_path, _LAYOUT_TABLE).body
assert "|" not in body
assert "Post one has enough words to keep\n\nPost two also has enough words here" in body
def test_data_table_stays_a_pipe_table(tmp_path: pathlib.Path) -> None:
"""A table whose cells hold only text and inline tags is one block."""
html = "<html><body><table><tr><th>Col one</th></tr><tr><td><b>Cell</b> value</td></tr></table></body></html>"
assert "| Col one |" in _convert(tmp_path, html).body
def test_sweep_is_deterministic(tmp_path: pathlib.Path) -> None:
"""Two conversions of one fixture give the same body and counts."""
first = _convert(tmp_path, _BOILERPLATE)
second = _convert(tmp_path, _BOILERPLATE)
assert first.body == second.body
assert first.warnings == second.warnings
def test_hidden_markers_drop(tmp_path: pathlib.Path) -> None:
"""``hidden``, ``aria-hidden``, ``visibility: hidden``, button, and select text is dropped."""
html = """<html><body>
<div hidden><p>SECRETA one two three four</p></div>
<div aria-hidden="TRUE"><p>SECRETB one two three four</p></div>
<div style="color: red; visibility:hidden"><p>SECRETC one two three four</p></div>
<button>SECRETD one two three four</button>
<select><option>SECRETE one two three four</option></select>
<p>Visible text with enough words here</p>
</body></html>"""
body = _convert(tmp_path, html).body
assert "SECRET" not in body
assert "Visible text with enough words here" in body
def test_chrome_roles_and_header(tmp_path: pathlib.Path) -> None:
"""Roles and a top-level header are chrome; a header inside article is not."""
html = """<html><body>
<header><p>Site banner text with many words</p></header>
<div role="Complementary"><p>Side widget text with many words</p></div>
<form><p>Form intro text with many words</p></form>
<article><header><p>Article byline with many words</p></header></article>
</body></html>"""
body = _convert(tmp_path, html).body
reasons = dict((text.strip(), reason) for reason, text in _reviews(body))
assert reasons["Site banner text with many words"] == "chrome"
assert reasons["Side widget text with many words"] == "chrome"
assert reasons["Form intro text with many words"] == "chrome"
assert "Article byline with many words" in _outside(body)
def test_text_folding_and_word_count(tmp_path: pathlib.Path) -> None:
"""``<b>Fa</b>st`` is one word and ``<br>`` separates words."""
joined = _convert(tmp_path, "<html><body><div><b>Fa</b>st go now</div></body></html>").body
assert _reviews(joined) == [("short", "**Fa**st go now")]
split = _convert(tmp_path, "<html><body><p>one<br>two three four</p></body></html>").body
assert _reviews(split) == []
def test_inline_card_holding_a_block_is_a_container(tmp_path: pathlib.Path) -> None:
"""``<a><div>...</div></a>`` is walked, so the card text is not one link run."""
html = '<html><body><a href="/card"><div>Card text with enough words inside it</div></a></body></html>'
body = _convert(tmp_path, html).body
assert "Card text with enough words inside it" in _outside(body)
def test_every_block_in_review_is_partial(tmp_path: pathlib.Path) -> None:
"""No keep and at least one review is partial with the second warning."""
result = _convert(tmp_path, "<html><body><p>Tiny</p></body></html>")
assert result.conversion == "partial"
assert "no block passed the content checks; every block is marked for review" in result.warnings
def test_empty_body_rule_runs_first(tmp_path: pathlib.Path) -> None:
"""A page with no text keeps the 7.1.5.4.5.5 warning and counts zero blocks."""
result = _convert(tmp_path, "<html><body><script>only</script><span> </span></body></html>")
assert result.conversion == "partial"
assert result.warnings[0].startswith("no text content")
assert "html sweep: 0 keep, 0 review, 0 drop" in result.warnings
def test_html_without_body_skips_head(tmp_path: pathlib.Path) -> None:
"""A fragment without ``<body>`` is swept without the head's title text."""
html = "<html><head><title>Only Title</title></head><p>Fragment text with enough words</p></html>"
result = _convert(tmp_path, html)
assert result.title == "Only Title"
assert result.body == "Fragment text with enough words\n"
def test_min_words_option(tmp_path: pathlib.Path) -> None:
"""``min_words`` decides when a loose block is short."""
html = "<html><body><div>Three word text</div></body></html>"
assert _reviews(_convert(tmp_path, html).body) == [("short", "Three word text")]
assert _reviews(_convert(tmp_path, html, min_words=2).body) == []
def test_settings_defaults_and_overrides(tmp_path: pathlib.Path) -> None:
"""SPEC 5.1: ``html.min_words`` and ``html.max_link_density`` validate their values."""
defaults = default_settings()
assert (defaults.html_min_words, defaults.html_max_link_density) == (4, 0.5)
good = load_ingest_settings(_wiki(tmp_path / "good", {"min_words": 6, "max_link_density": 1}))
assert (good.html_min_words, good.html_max_link_density) == (6, 1.0)
for index, bad in enumerate((0, True, "4", 1.5)):
loaded = load_ingest_settings(_wiki(tmp_path / f"words{index}", {"min_words": bad}))
assert loaded.html_min_words == 4, bad
for index, bad in enumerate((True, "0.5", 1.5, -0.1)):
loaded = load_ingest_settings(_wiki(tmp_path / f"density{index}", {"max_link_density": bad}))
assert loaded.html_max_link_density == 0.5, bad
zero = load_ingest_settings(_wiki(tmp_path / "zero", {"max_link_density": 0}))
assert zero.html_max_link_density == 0.0
odd = _wiki(tmp_path / "odd")
settings = json.loads((odd / "fiona-settings.json").read_text(encoding="utf-8"))
settings.setdefault("fiona", {}).setdefault("wiki", {})["ingest"] = {"html": "nope"}
(odd / "fiona-settings.json").write_text(json.dumps(settings), encoding="utf-8")
assert load_ingest_settings(odd).html_min_words == 4
def test_registry_hands_settings_to_html(tmp_path: pathlib.Path) -> None:
"""``default_registry(settings)`` builds the html converter with those values."""
settings = default_settings()
settings.html_min_words = 9
converter = default_registry(settings).find("text/html", ".html")
assert isinstance(converter, HtmlConverter)
assert converter.min_words == 9
assert default_registry().find("text/html", "").min_words == 4
def test_pipeline_uses_wiki_min_words(tmp_path: pathlib.Path) -> None:
"""A wiki with ``min_words: 2`` keeps a three-word div through the full pipeline."""
root = _wiki(tmp_path, {"min_words": 2})
source = tmp_path / "three.html"
source.write_text("<html><body><div>Three word text</div></body></html>", encoding="utf-8")
result = default_pipeline(root).run(str(source))
assert result.exit_code == 0
page = (root / result.page_path).read_text(encoding="utf-8")
assert "Three word text" in page
assert "fiona:review" not in page
def test_review_markers_pass_wiki_sanity(tmp_path: pathlib.Path) -> None:
"""An ingested page with review blocks keeps its markers and passes markdown lint."""
root = _wiki(tmp_path)
source = tmp_path / "sweep-page.html"
source.write_text(_BOILERPLATE, encoding="utf-8")
result = default_pipeline(root).run(str(source))
assert result.exit_code == 0
page = (root / result.page_path).read_text(encoding="utf-8")
assert len(_reviews(page)) == 5
assert not any("markdown style violations" in warning for warning in result.warnings)
results = run_wiki_checks(root)
hits = [
violation
for check in results["checks"].values()
for violation in check.get("violations", [])
if "sweep-page.md" in str(violation)
]
assert [hit for hit in hits if "MD" in str(hit)] == []
@pytest.mark.parametrize("fixture", [_LANDING, _BOILERPLATE, _ENCYCLOPEDIA, _QA, _FORUM, _LAYOUT_TABLE])
def test_markers_sit_on_their_own_lines(tmp_path: pathlib.Path, fixture: str) -> None:
"""Every marker is a whole line and opens and closes in pairs."""
lines = _convert(tmp_path, fixture).body.splitlines()
opens = [line for line in lines if "fiona:review" in line and "/fiona:review" not in line]
closes = [line for line in lines if "/fiona:review" in line]
assert all(re.fullmatch(r"<!-- fiona:review (chrome|repeat|links|short) -->", line) for line in opens)
assert all(line == "<!-- /fiona:review -->" for line in closes)
assert len(opens) == len(closes)