From de28441943e4e9141a33c04457d970b80d222205 Mon Sep 17 00:00:00 2001 From: nvaldez Date: Mon, 19 Oct 2026 09:12:04 +0200 Subject: [PATCH] feat: sync Nokia fork product through SPEC 1.18 (v1.16.0) MEMORY.md and skill memory template, ingest auth and html_sweep, TOC slug fix, run_tests CI output, and matching tests. Excludes GitLab CI and publishing docs. --- .gitignore | 4 + CHANGELOG.md | 195 ++++++ SPEC.md | 654 +++++++++++++++--- pyproject.toml | 2 +- src/fiona/data/default.gitignore | 4 + src/fiona/data/llm_wiki/AGENTS.md | 12 +- src/fiona/data/llm_wiki/MEMORY.md | 179 +++++ .../data/llm_wiki/constellation/colorful.json | 6 +- .../data/llm_wiki/constellation/dark.json | 6 +- .../llm_wiki/constellation/grayscale.json | 6 +- .../llm_wiki/constellation/night-sky.json | 6 +- .../llm_wiki/constellation/professional.json | 6 +- src/fiona/data/llm_wiki/fiona-settings.json | 6 +- .../data/llm_wiki/utils/skills/README.md | 10 +- .../data/llm_wiki/utils/skills/fiona/SKILL.md | 77 ++- .../llm_wiki/utils/skills/memory/MEMORIES.md | 173 +++++ .../llm_wiki/utils/skills/memory/SKILL.md | 109 +++ .../sectionize-markdown.md | 157 +++++ src/fiona/librarian/toc_generator.py | 7 +- src/fiona/run_tests.py | 46 +- src/fiona/wiki/ingest/converters/__init__.py | 16 +- src/fiona/wiki/ingest/converters/html.py | 53 +- .../wiki/ingest/converters/html_sweep.py | 285 ++++++++ src/fiona/wiki/ingest/fetchers.py | 46 +- src/fiona/wiki/ingest/log_writer.py | 2 +- src/fiona/wiki/ingest/models.py | 18 + src/fiona/wiki/ingest/pipeline.py | 64 +- src/fiona/wiki/ingest/settings.py | 46 ++ src/fiona/wiki/templates/graph_physics.js | 13 +- src/fiona/wiki/update_manager.py | 11 +- tests/test_bootstrap_install.py | 13 +- tests/test_gittool.py | 29 + tests/test_graph_generator.py | 70 +- tests/test_html_sweep.py | 399 +++++++++++ tests/test_ingest.py | 20 + tests/test_ingest_auth.py | 350 ++++++++++ tests/test_ingest_edges.py | 16 +- tests/test_run_tests.py | 2 +- tests/test_template_docs.py | 114 +++ tests/test_toc_generator.py | 9 +- tests/test_update_manager.py | 37 + todo.md | 479 ++++++++++++- 42 files changed, 3566 insertions(+), 191 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 src/fiona/data/llm_wiki/MEMORY.md create mode 100644 src/fiona/data/llm_wiki/utils/skills/memory/MEMORIES.md create mode 100644 src/fiona/data/llm_wiki/utils/skills/memory/SKILL.md create mode 100644 src/fiona/data/llm_wiki/utils/skills/sectionize-markdown/sectionize-markdown.md create mode 100644 src/fiona/wiki/ingest/converters/html_sweep.py create mode 100644 tests/test_html_sweep.py create mode 100644 tests/test_ingest_auth.py diff --git a/.gitignore b/.gitignore index 1d3084a..9c4f262 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,10 @@ htmlcov/ # Windows Thumbs.db +# Obsidian per-user vault state +.obsidian/workspace*.json +.obsidian/cache + # Python *.pyc *.pyo diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..271e4c2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,195 @@ +--- +name: changelog +version: 1.2.0 +title: Changelog +created: 2026-10-08 +updated: 2026-10-15 +type: summary +--- + +# Changelog + +Revisions of [SPEC](./SPEC.md), newest first. Each entry names the +requirements it adds or changes. "Unbuilt" means the requirement is +specified and marked `NOT_IMPLEMENTED`. + +## 1.18.0 - 2026-10-15 + +- Adds skill memory to the template (*3.5.*): the `memory` skill + under `utils/skills/memory/` says how a skill keeps its own + `MEMORIES.md` beside its `SKILL.md`, and ships the blank + `MEMORIES.md` to copy. Wiki-wide preferences stay in `MEMORY.md`. + +## 1.17.0 - 2026-10-15 + +- The `.gitignore` written by `wiki init` ignores Obsidian's per-user + vault state, `.obsidian/workspace*.json` and `.obsidian/cache`, so a + wiki opened as an Obsidian vault stays clean between saves. Shared + vault settings such as `.obsidian/app.json` stay tracked (*3.2.3.*). + +## 1.16.0 - 2026-10-14 + +- Adds the agent memory `MEMORY.md` to the template (*3.4.*): a + sectionized file of owner preferences, corrections, and verified + facts, linked from `AGENTS.md`. `AGENTS.md` says to update it on + an important memory or when the user explicitly asks (*3.4.6.*). `wiki update` adds it when missing + and never compares or replaces it after that (*3.4.4.*). + +## 1.14.1 - 2026-10-08 + +- Moves the revision history out of the spec into this file. + +## 1.14.0 - 2026-10-08 + +- Stages raw pages in a `refine/` directory at the wiki root, and has + the agent move each refined page into `wiki/` (*7.1.5.15.*). + Unbuilt. + +## 1.13.0 - 2026-10-07 + +- Builds login detection with exit 4 (*7.1.5.2.8.*), `--origin` + (*7.1.5.2.9.*), the agent steps for exit 4 (*7.1.5.14.*), and the + HTML block sweep (*7.1.5.4.5.6.*). + +## 1.11.0 + +- Specifies `wiki search` (*4.7.7.*, *7.1.6.*), a keyword search that + prints `:: ` and shortens long lines around the + match. Unbuilt. + +## 1.10.0 + +- Replaces the first-`
` content root of the `html` converter + with a block sweep that scores every block as `keep`, `review`, or + `drop` (*7.1.5.4.5.6.*). Unbuilt. + +## 1.9.0 + +- `wiki ingest` stops with exit 4 on a URL that needs a login + (*7.1.5.2.8.*). Unbuilt. +- Adds `--origin` for a file saved by hand (*7.1.5.2.9.*). Unbuilt. +- Tells the agent how to get the source instead (*7.1.5.14.*). + Unbuilt. + +## 1.8.0 + +Adds the findings of the 2026-10-02 manual test. All unbuilt: + +- One-line CLI errors with no traceback (*1.2.7.*, *7.1.1.4.*). +- `--version` (*2.3.*). +- A quiet default stderr (*7.17.3.*). +- An ingest hint for a drop-zone file that already moved + (*7.1.5.2.7.*). +- A dry run that does not log writes (*7.1.5.8.1.*). +- A lint-clean ingest log line (*7.1.5.6.1.*). +- A follow-up hint for style violations left on a page + (*7.1.5.5.6.*). +- Single-file `mdoctor` (*7.4.4.*). +- Compact housekeeping sanity details (*7.2.5.1.*). +- A quiet graph overwrite (*7.1.4.5.9.1.*). +- A usage section in the README (*7.18.4.*). + +## 1.7.0 + +- `librarian move` rewrites links to the moved paths across the whole + wiki (*7.3.3.6.*). + +## 1.6.0 + +- Specifies `librarian move` (*7.3.3.*). Unbuilt. + +## 1.5.2 + +- Gives `table_of_contents.md` frontmatter `name` and `version` + (*7.3.1.1.1.*), so `fmlint` accepts it. + +## 1.5.0 + +- Omits markdown files matched by `fiona.wiki.graph.ignore_files` + from the graph (*7.1.4.20.*). The shipped list is + `*/table_of_contents.md`, because a table of contents links to + every page. + +## 1.4.0 + +- Names the core-file record `corefiles.json` (*7.1.2.14.*). +- Skips paths in `fiona.wiki.update.ignore_files` (*7.1.2.15.*). The + shipped list names `corefiles.json`. Only `wiki update` writes that + file. + +## 1.3.0 + +- Specifies directory nodes (*7.1.4.18.*) and `wiki graph --serve` + (*7.1.4.19.*). Both unbuilt. + +## 1.2.0 + +- Folds text line endings before the SHA-256 compare (*7.1.2.13.*). +- Writes `corefiles.json`, so a later update does not list a core + file whose version and hashes are unchanged (*7.1.2.14.*). + +## 1.1.0 + +- Treats a differing `fiona-settings.json` as a conflict and writes + `fiona-settings-update.json` (*7.1.2.3.*). + +## 1.0.0 + +- Groups each module's functions under that module. +- Moves Non-Goals, Priority Order, and Open Questions to Appendices + A, B, and C. + +## 0.99.0 + +- Builds `wiki ingest` (*7.1.5.*). + +## 0.98.0 + +- Specifies that `wiki graph` omits a file matched by `.gitignore` + (*7.1.4.17.*). Unbuilt. + +## 0.97.0 + +- Rejects a module-level assignment that follows the first function + or class (*6.4.3.*). + +## 0.96.0 + +- Compares a file without a version by SHA-256 and shows both short + hashes in the update report (*7.1.2.12.*). + +## 0.94.0 + +- `wiki update` yields to the owner's markdown and writes + `-update.md` instead (*7.1.2.11.*). +- Adds `fiona-assistant housekeeping` (*7.2.*). + +## 0.93.0 + +- Adds SHA-256 reuse across `raw/`, the complete `ingest` + frontmatter map, and the agent metadata check. + +## 0.92.0 + +- Specifies `wiki ingest` (*7.1.5.*). Unbuilt. + +## 0.90.0 + +- Specifies `node_importance_min_point` (*7.1.4.16.*). Unbuilt. + +## 0.41.0 + +- Installs one `fiona` package and keeps tests and repo scripts + outside it. + +## 0.40.4 + +- Adds the PYC001 exception (*6.4.2.*). + +## 0.40.1 + +- Numbers nested requirements such as `7.14.1.1`. + +## 0.40.0 + +- Reorganizes the sheet into contracts. diff --git a/SPEC.md b/SPEC.md index f43e755..e9d4927 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,9 +1,9 @@ --- name: Fiona-Assistant Specification -version: 1.9.1 +version: 1.18.0 title: Project Spec Sheet created: 2026-09-23 -updated: 2026-10-06 +updated: 2026-10-15 type: summary --- @@ -19,6 +19,8 @@ type: summary - [3.1. What Counts as a Wiki](#31-what-counts-as-a-wiki) - [3.2. Core Files and Read-only Zones](#32-core-files-and-read-only-zones) - [3.3. Deployed Wiki Rules](#33-deployed-wiki-rules) + - [3.4. Agent Memory](#34-agent-memory) + - [3.5. Skill Memory](#35-skill-memory) - [4. Command Catalog](#4-command-catalog) - [4.1. librarian](#41-librarian) - [4.1.1. librarian toc](#411-librarian-toc) @@ -35,6 +37,7 @@ type: summary - [4.7.4. wiki sanity](#474-wiki-sanity) - [4.7.5. wiki graph](#475-wiki-graph) - [4.7.6. wiki ingest](#476-wiki-ingest) + - [4.7.7. wiki search](#477-wiki-search) - [4.8. housekeeping](#48-housekeeping) - [4.9. Repository runners](#49-repository-runners) - [5. Settings Schema](#5-settings-schema) @@ -55,6 +58,7 @@ type: summary - [7.1.3. wiki sanity](#713-wiki-sanity) - [7.1.4. wiki graph](#714-wiki-graph) - [7.1.5. wiki ingest](#715-wiki-ingest) + - [7.1.6. wiki search](#716-wiki-search) - [7.2. housekeeping](#72-housekeeping) - [7.3. librarian](#73-librarian) - [7.3.1. Folder and In-file TOC](#731-folder-and-in-file-toc) @@ -90,58 +94,9 @@ this file, and no source tree, must be able to produce the same observable software. Status values are `Implemented`, `Partially Implemented`, `Planned`, and `Won't Do`. -Revision 0.40.0 reorganizes the sheet into contracts. Revision 0.40.1 -numbers nested requirements such as `7.14.1.1`. Revision 0.40.4 adds -the PYC001 exception in *6.4.2.*. Revision 0.41.0 installs one -`fiona` package and keeps tests and repo scripts outside it. -Revision 0.90.0 specifies `node_importance_min_point` in *7.1.4.16.* -and leaves that requirement unbuilt. Revision 0.92.0 specifies -`wiki ingest` in *7.1.5.* and leaves it unbuilt. Revision 0.93.0 adds -SHA-256 reuse across `raw/`, the complete `ingest` frontmatter map, -and the agent metadata check. Revision 0.94.0 makes `wiki update` -yield to the owner's markdown and write `-update.md` instead -(*7.1.2.11.*), and adds `fiona-assistant housekeeping` (*7.2.*). -Revision 0.96.0 compares a file without a version by SHA-256 and -shows both short hashes in the update report (*7.1.2.12.*). -Revision 0.97.0 rejects a module-level assignment that follows the -first function or class (*6.4.3.*). Revision 0.98.0 specifies that -`wiki graph` omits a file matched by `.gitignore` (*7.1.4.17.*) and -leaves that requirement unbuilt. Revision 0.99.0 builds `wiki ingest` -(*7.1.5.*). Revision 1.0.0 groups each module's functions under that -module and moves Non-Goals, Priority Order, and Open Questions -to Appendices A, B, and C. Revision 1.1.0 treats a differing -`fiona-settings.json` as a conflict and writes -`fiona-settings-update.json` (*7.1.2.3.*). Revision 1.2.0 folds -text line endings before the SHA-256 compare (*7.1.2.13.*) and -writes `corefiles.json` so a later update does not list a -core file whose version and hashes are unchanged (*7.1.2.14.*). -Revision 1.3.0 specifies directory nodes (*7.1.4.18.*) and -`wiki graph --serve` (*7.1.4.19.*) and leaves both unbuilt. -Revision 1.4.0 names the core-file record `corefiles.json` -(*7.1.2.14.*) and skips paths in `fiona.wiki.update.ignore_files` -(*7.1.2.15.*). The shipped list names `corefiles.json`. Only -`wiki update` writes that file. Revision 1.5.0 omits markdown -files matched by `fiona.wiki.graph.ignore_files` from the graph -(*7.1.4.20.*). The shipped list is `*/table_of_contents.md`, -because a table of contents links to every page. Revision 1.5.2 -gives that file frontmatter `name` and `version` (*7.3.1.1.1.*) -so `fmlint` accepts it. Revision 1.6.0 specifies `librarian move` -(*7.3.3.*) and leaves it unbuilt. Revision 1.7.0 makes that -command rewrite links to the moved paths across the whole wiki -(*7.3.3.6.*). Revision 1.8.0 adds the findings of the 2026-10-02 -manual test and leaves them unbuilt: one-line CLI errors with no -traceback (*1.2.7.*, *7.1.1.4.*), `--version` (*2.3.*), a quiet -default stderr (*7.17.3.*), an ingest hint for a drop-zone file that -already moved (*7.1.5.2.7.*), a dry run that does not log writes -(*7.1.5.8.1.*), a lint-clean ingest log line (*7.1.5.6.1.*), a -follow-up hint for style violations left on a page (*7.1.5.5.6.*), -single-file `mdoctor` (*7.4.4.*), compact housekeeping sanity details -(*7.2.5.1.*), a quiet graph overwrite (*7.1.4.5.9.1.*), and a usage -section in the README (*7.18.4.*). Revision 1.9.0 makes -`wiki ingest` stop with exit 4 on a URL that needs a login -(*7.1.5.2.8.*), adds `--origin` for a file saved by hand -(*7.1.5.2.9.*), and tells the agent how to get the source instead -(*7.1.5.14.*). All three are unbuilt. +The revision history of this document is in +[CHANGELOG](./CHANGELOG.md). Add an entry there, newest first, each +time the `version` above changes. Each feature section starts with a `Status` line. Requirements use `- *N.M.K.*`. Each `*acceptance*` sits under the requirement it tests. @@ -315,6 +270,7 @@ the text `fiona.llm-wiki.ignore_files must be a list of strings`. ```text AGENTS.md +MEMORY.md README.md fiona-settings.json constellation/colorful.json @@ -337,11 +293,15 @@ utils/skills/fiona/SKILL.md utils/skills/fiona/markdown_rules.md utils/skills/fiona/llm-wiki-mod.md utils/skills/python-tdd/SKILL.md +utils/skills/memory/SKILL.md +utils/skills/memory/MEMORIES.md utils/personas/default/SOUL.md utils/personas/template/SOUL.md utils/skills_local/README.md ``` +`NOT_IMPLEMENTED` `refine/README.md` joins this tree (*7.1.5.15.10.*). + The shipped `fiona-settings.json` is the default document in section 5.1. `wiki/schema.md` is the page-type and tag taxonomy. This spec's frontmatter table in section 6.2 is the field contract; @@ -370,6 +330,7 @@ Core directories (every file under them is core): - `raw/` - `utils/` - `constellation/` +- `NOT_IMPLEMENTED` `refine/` (*7.1.5.15.10.*) Read-only zones, including the zone directory itself: @@ -381,6 +342,10 @@ Read-only zones, including the zone directory itself: `wiki/`. The only file it writes there is an update file for a template page that is newer than the wiki's copy (*7.1.2.11.*). +`MEMORY.md` belongs to the wiki once it is there. `wiki update` adds +it when it is missing and never compares or replaces it after that +(*3.4.4.*). + `wiki update` adds missing core files and directories. It never overwrites an existing markdown file: when the template carries a newer version, it writes `-update.md` beside the file and @@ -403,7 +368,10 @@ After the copy, `wiki init` creates a git repository and writes `.gitignore` (the same patterns, including `__pycache__/`, `*.pyc`, and `*fiona.log`). When this package is not the source checkout, that text is the packaged `default.gitignore`. The file adds -`personal/` when that pattern is missing. +`personal/` when that pattern is missing. The set also ignores +Obsidian's per-user vault state, `.obsidian/workspace*.json` and +`.obsidian/cache`, so a wiki opened as an Obsidian vault stays +clean between saves (*3.2.3.*). An existing `.gitignore` keeps every line it already has; missing patterns from that set are appended. Init then stages everything and commits `Initial llm-wiki commit`. Template paths that match @@ -420,6 +388,11 @@ the copy still counts as success and stdout adds message names each conflicting indicator, and writes nothing. - *acceptance*: An existing `README.md` alone is enough to refuse, and that file's bytes are unchanged. +- *3.2.3.* The `.gitignore` written by `wiki init` ignores Obsidian's + per-user state and keeps the shared vault settings tracked. + - *acceptance*: After `wiki init`, `git check-ignore` matches + `.obsidian/workspace.json`, `.obsidian/workspace-mobile.json`, + and `.obsidian/cache`, and does not match `.obsidian/app.json`. ### 3.3. Deployed Wiki Rules @@ -450,6 +423,86 @@ this repository only. `DATE_PLACEHOLDER`. - *acceptance*: The log body has exactly one bullet. +### 3.4. Agent Memory + +> Status: Implemented + +`MEMORY.md` at the wiki root is the agent's memory of how the owner +wants this wiki run: standing preferences, corrections, repeated +request patterns, and facts a run verified about the wiki's tools +and setup. Domain knowledge goes to pages under `wiki/`, not here. +The template ships the rules for the file and empty categories. +From then on the file grows with the wiki it belongs to. + +- *3.4.1.* The template ships `MEMORY.md` at its root with + frontmatter `name: memory` and a `version`. `wiki init` copies it + with the rest of the tree. + - *acceptance*: The template `MEMORY.md` exists and its + frontmatter has `name: memory` and `version`. +- *3.4.2.* `MEMORY.md` is sectionized as the template's + `utils/skills/sectionize-markdown/sectionize-markdown.md` + describes. Every heading below the title is numbered. Every list + item outside the TOC and code blocks starts with a bold id that + extends the number of its section. No id repeats. + - *acceptance*: A scan of the template `MEMORY.md` finds no + unnumbered heading, no list item without an id, no id outside + its section, and no duplicate id. +- *3.4.3.* Section 1 of `MEMORY.md` says when to read it, when to + write an entry, how to write one, and how entries evolve. An entry + uses the knowledge-entry shape of sectionize-markdown section 4. + An entry never overrides the boundaries of the template + `AGENTS.md`. + - *acceptance*: `MEMORY.md` has the subsections + `1.1 When to read`, `1.2 When to write`, `1.3 How to write`, and + `1.4 How the memory evolves`. +- *3.4.4.* `MEMORY.md` evolves with the wiki, not with the template. + `wiki update` adds it when the wiki has none. Once the wiki has + the file, check-update, `wiki update`, and the `corefiles.json` + record leave it out, whatever the template version: it is never + compared, replaced, or written as `MEMORY-update.md`. A match in + `fiona.wiki.update.ignore_files` (*7.1.2.15.*) also stops the + add. + - *acceptance*: A template `MEMORY.md` with a higher version than + the wiki copy is absent from check-update. `wiki update --yes` + leaves the wiki copy byte-for-byte unchanged, writes no + `MEMORY-update.md`, and `corefiles.json` has no `MEMORY.md` + entry. + - *acceptance*: A wiki without `MEMORY.md` lists it under files to + add. +- *3.4.5.* The template `AGENTS.md` links `./MEMORY.md` and tells + the agent to read it once the Fiona skill is loaded. + - *acceptance*: The relative link appears in that file. +- *3.4.6.* The template `AGENTS.md` tells the agent to update + `MEMORY.md` whenever an important memory comes up, and whenever + the user explicitly asks it to remember something. + - *acceptance*: That file says to update `MEMORY.md` on an + important memory and when the user explicitly asks. + +### 3.5. Skill Memory + +> Status: Implemented + +A skill can keep a memory of its own: what its users want from that +skill, kept across sessions. The file is `MEMORIES.md` beside the +skill's `SKILL.md`. The template ships the instructions and a blank +file under `utils/skills/memory/`, so an agent asked to create a skill +with memory has both. Wiki-wide preferences stay in `MEMORY.md` +(*3.4.*). + +- *3.5.1.* The template ships `utils/skills/memory/SKILL.md` with + frontmatter `name: memory` and a `version`. It says what a skill + memory holds, how to give a skill one, and how the skill's + `SKILL.md` loads it. + - *acceptance*: The file exists, its frontmatter has + `name: memory` and `version`, and it links `./MEMORIES.md` and + the wiki root `MEMORY.md`. +- *3.5.2.* The template ships `utils/skills/memory/MEMORIES.md`, a + blank memory for one skill, with frontmatter `name` and `version`. + It is sectionized as *3.4.2.* describes, and its section 1 has the + subsections of *3.4.3.*. + - *acceptance*: The same scans as *3.4.2.* and *3.4.3.* pass on + that file. + --- ## 4. Command Catalog @@ -859,7 +912,7 @@ fiona-assistant wiki ingest --pending [--wiki DIR] [--json] `Error: Directory '' is not an llm-wiki directory` and exit 1. `` and `--pending` are mutually exclusive, and one of them is required. argparse reports a violation and exits 2. -`NOT_IMPLEMENTED` `--origin URL` names the URL a local file was saved +`--origin URL` names the URL a local file was saved from (*7.1.5.2.9.*). Human stdout on success, the lines that apply, in this order: @@ -895,7 +948,7 @@ relative to the wiki root and use `/`: ```json { "source": "", - "origin": "", + "origin": "", "origin_kind": "url", "media_type": "application/pdf", "raw_path": "raw/papers/.pdf", @@ -917,6 +970,8 @@ is the page that already cites the matching raw copy, else `null` (*7.1.5.3.5.*). `--pending --json` prints `{"pending": [{"page_path": ..., "sources": [...], "origin": ...}], "total": }`. +`NOT_IMPLEMENTED` Once staging is built, every page path above is +`refine/` (*7.1.5.15.2.*). | Exit | Meaning | | --- | --- | @@ -924,7 +979,7 @@ prints `{"pending": [{"page_path": ..., "sources": [...], | 1 | Error on stderr. Nothing written under the wiki. | | 2 | argparse usage error. | | 3 | Raw copy archived. No page: no converter, or conversion failed. | -| 4 | `NOT_IMPLEMENTED` The URL needs a login (*7.1.5.2.8.*). Error on stderr. Nothing written under the wiki. | +| 4 | The URL needs a login (*7.1.5.2.8.*). Error on stderr. Nothing written under the wiki. | Errors, each on stderr with exit 1: @@ -944,7 +999,7 @@ Error: --origin needs a local file: Error: --origin must be an http or https URL: ``` -`NOT_IMPLEMENTED` A URL that needs a login prints two lines on stderr +A URL that needs a login prints two lines on stderr and exits 4 (*7.1.5.2.8.*): ```text @@ -960,6 +1015,60 @@ A failed conversion prints `Error: Conversion failed: : ` on stderr and exits 3, not 1, because the raw copy stays. `NOT_IMPLEMENTED` The `(moved to ...)` form is *7.1.5.2.7.*. +#### 4.7.7. wiki search + +`NOT_IMPLEMENTED` The contract is section 7.1.6. + +```text +fiona-assistant wiki search [ ...] [--wiki DIR] + [--all-words] [--case-sensitive] [--include GLOB] [--json] +``` + +`--wiki` is the wiki root and defaults to `.`. It must contain +`fiona-settings.json`, else stderr +`Error: Directory '' is not an llm-wiki directory` and exit 1. +At least one keyword is required, else argparse exits 2. A quoted +phrase such as `"agent harness"` is one keyword. + +Human stdout, one line per matching line, then one summary line: + +```text +:: +... more matches. Narrow the keywords or raise max_results. +Matches: lines in files +``` + +`` is relative to the wiki root and uses `/`. `` starts +at 1. The `path:line:` form is the one grep prints, so editors and +terminals open it as a link. The `... more matches` line appears +only when the result limit cut the list. + +`--json` prints one object and nothing else on stdout: + +```json +{ + "keywords": ["harness"], + "matches": [ + {"path": "wiki/sources/agent-harness.md", "line": 44, + "text": "... how LangSmith agent harness works ...", + "columns": [21]} + ], + "total": 1, + "files": 1, + "truncated": false +} +``` + +`text` is the shortened text of *7.1.6.4.* `columns` holds the +1-based column of each match in the full line. `total` counts +every matching line, including those past the limit. + +| Exit | Meaning | +| --- | --- | +| 0 | Search ran. Zero matches is still exit 0. | +| 1 | Error on stderr. | +| 2 | argparse usage error. | + ### 4.8. housekeeping `fiona-assistant housekeeping [directory] [--json]`. One call runs @@ -1103,7 +1212,7 @@ Shipped document (logical content; `wiki init` copies the file): "consolidate_external_links": false, "consolidate_external_links_depth": 0, "consolidate_external_links_max_char": 8, - "ignore_files": ["*/table_of_contents.md"] + "ignore_files": ["*/table_of_contents.md", "*/README.md", "*/AGENTS.md"] }, "update": { "ignore_files": ["corefiles.json"] @@ -1175,6 +1284,10 @@ Shipped document (logical content; `wiki init` copies the file): | `fiona.wiki.ingest.default_target` | string | `sources` | wiki ingest | | `fiona.wiki.ingest.raw_categories` | object of strings | `{}` | wiki ingest | | `fiona.wiki.ingest.login_url_patterns` | list of regex strings | see below | wiki ingest | +| `fiona.wiki.ingest.html.min_words` | int `>= 1` | 4 | wiki ingest | +| `fiona.wiki.ingest.html.max_link_density` | number `0..1` | 0.5 | wiki ingest | +| `fiona.wiki.search.max_line_chars` | int `>= 20` | 160 | wiki search | +| `fiona.wiki.search.max_results` | int `>= 0` | 200 | wiki search | Bool pymarkdown properties use `set_boolean_property`, ints use `set_integer_property`, strings use `set_string_property`, applied as @@ -1259,10 +1372,18 @@ name to a directory under `raw/`, such as `{"text": "transcripts"}`. A value that is absent, of the wrong type, or out of range uses the default in the table. The packaged `fiona-settings.json` omits `fiona.wiki.ingest`. Absent keys use the defaults in the table. -`login_url_patterns` is read by *7.1.5.2.8.3.* and is not built -yet. A list replaces the default list, it does not extend it. An -entry that does not compile is skipped with one warning. The -default list: +`html.min_words` and `html.max_link_density` are read by the block +sweep (*7.1.5.4.5.6.*). + +`fiona.wiki.search` is read by section 7.1.6 and is not built yet. +`max_line_chars` is the longest `` a match line prints. +`max_results` is how many match lines print. `0` prints them all. +A value that is absent, of the wrong type, or out of range uses the +default in the table. +`login_url_patterns` is read by *7.1.5.2.8.3.* A list replaces +the default list, it does not extend it. An entry that does not +compile, or is not a string, is skipped with one warning. A value +that is not a list uses the default list: ```json [ @@ -1367,6 +1488,8 @@ buttons, and a quiet radius. | `highlight.edge_color` | Edge color while a node is highlighted | | `highlight.size_multiplier` | Radius multiplier while highlighted | | `clicked.node_color` | Node color after a click | +| `orphan.enabled` | When true, orphan nodes use `orphan.node_color` | +| `orphan.node_color` | Fill for an orphan node (*7.1.4.2.2.*) | | `buttons.background/text/hover/disabled` | Button colors | | `card.background/border` | Cards, legend, and the control bar | | `shapes.page` | `circle`, `cloud`, `star`, or `folder` | @@ -1415,6 +1538,18 @@ A theme that sets only some of these keeps the rest from shape. A `constellation-settings.json` beside it replaces `background` with its own value. A file of `{` does not remove `colorful`. +- *5.3.5.* A theme can paint orphan nodes in one color. When + `orphan.enabled` is true, an orphan page (*7.1.4.2.2.*) is filled + with `orphan.node_color` instead of the `node_colors` ramp. Hover + and click colors still replace it. External-link and source nodes + are never orphans. Every packaged theme has an `orphan` object + with `enabled` false, so the graph looks the same until a theme + turns it on. + - *acceptance*: Every resolved packaged theme has + `orphan.enabled` false and an `orphan.node_color`. A local + `constellation/dark.json` setting `orphan.enabled` true keeps + the packaged `orphan.node_color`. The graph HTML reads + `theme.orphan.enabled` and `theme.orphan.node_color`. --- @@ -2318,9 +2453,11 @@ used. list. A value that is not a list of strings raises `ValueError` with the text `fiona.wiki.graph.ignore_files must be a list of strings`. A pattern that starts with `*/` also matches that same path at - the scan root. The shipped list is `*/table_of_contents.md`. - A table of contents links to every page, so leaving it in the - graph hides the real links. + the scan root. The shipped list is `*/table_of_contents.md`, + `*/README.md`, and `*/AGENTS.md`. A table of contents links to + every page, so leaving it in the graph hides the real links. + A README or AGENTS file describes the tree, not the wiki's + knowledge, so it is left out too. - *acceptance*: With that pattern, `table_of_contents.md` at the scan root and `docs/table_of_contents.md` are absent from JSON `nodes`, `edges`, and `orphans`. `a.md` and `b.md` stay. @@ -2585,7 +2722,7 @@ and one job. Class names are in Appendix D. | Fetch | the `` argument | a local file, its origin, and its media type | | Archive | that local file | a file under `raw/` | | Convert | the archived file | markdown body, title, metadata, conversion state, warnings | -| Write | the conversion | one page under `wiki/` with `status: raw` | +| Write | the conversion | one page under `wiki/` with `status: raw` (under `refine/` once *7.1.5.15.* is built) | | Record | the run | one line appended to `wiki/log.md` | Shipped converters, in registration order: @@ -2673,7 +2810,7 @@ Candidates for later formats are in Appendix C. `.\ingest\a.md` exits 0, then exits 1 with `Error: Source not found: .\ingest\a.md (moved to raw/articles/a.md by an earlier ingest)` on one line. - - *7.1.5.2.8.* `NOT_IMPLEMENTED` A URL that needs a login is an + - *7.1.5.2.8.* A URL that needs a login is an error, not a source. The tool cannot log in, so it stops and leaves the next step to the agent (*7.1.5.14.*). A login page must never reach `raw/` as if it were the source. @@ -2723,7 +2860,7 @@ Candidates for later formats are in Appendix C. with no redirect and no password input, is ingested with exit 0. - *acceptance*: A 404 still exits 1 with `Download failed`. - - *7.1.5.2.9.* `NOT_IMPLEMENTED` `--origin URL` records where a + - *7.1.5.2.9.* `--origin URL` records where a local file came from, so a source saved by hand after *7.1.5.2.8.* keeps its real provenance. - *7.1.5.2.9.1.* `--origin` is valid only when `` is a @@ -2749,6 +2886,19 @@ Candidates for later formats are in Appendix C. `https://host/a`. No network call is made. - *acceptance*: `wiki ingest https://host/a --origin https://host/b` exits 1 and makes no network call. + - *7.1.5.2.10.* A local file inside the wiki root is recorded by + its path relative to the wiki root, with `/` separators, e.g. + `ingest/notes.txt`. The wiki can be cloned anywhere, so an + absolute path inside it is wrong on every other machine. A file + outside the wiki root keeps its resolved absolute path. This + `origin` is the same in the page frontmatter, the JSON result, + and the log bullet (*7.1.5.6.*). `--origin` replaces it + (*7.1.5.2.9.2.*). + - *acceptance*: `wiki ingest /ingest/a.txt`, given as an + absolute path, writes `origin: ingest/a.txt` and + `origin_kind: file`, and the log bullet names + `ingest/a.txt`. A file under the system temp directory, + outside the wiki, keeps its absolute path. - *7.1.5.3.* Archive. - *7.1.5.3.1.* The default raw path is `raw//.`. The category is the raw category @@ -2831,12 +2981,13 @@ Candidates for later formats are in Appendix C. - *7.1.5.4.5.* `html` parses with beautifulsoup4 and converts with markdownify. - *7.1.5.4.5.1.* The title is ``, then `og:title`, then the - first `<h1>`. `source_meta` holds `lang`, the canonical link, + first `<h1>` in `<body>` that has text. `source_meta` holds `lang`, the canonical link, and the `description`, `author`, `og:*`, and `article:published_time` meta values that are present. - - *7.1.5.4.5.2.* The content root is the first `<article>`, then - `<main>`, then `<body>`. `script`, `style`, `noscript`, - `template`, and comments are dropped. + - *7.1.5.4.5.2.* `script`, `style`, `noscript`, `template`, and + comments are dropped. The content is `<body>`, read by the + block sweep in *7.1.5.4.5.6.* A `template` that has a + `shadowrootmode` attribute is kept (*7.1.5.4.5.6.1.*). - *7.1.5.4.5.3.* A relative `href` or `src` is made absolute against `<base href>`, then the origin URL. A local file has no origin URL, so its relative links are left to *7.1.5.5.4.*. @@ -2845,6 +2996,128 @@ Candidates for later formats are in Appendix C. - *7.1.5.4.5.5.* A body with no text outside markup is `partial` with the warning `no text content; the page may need JavaScript or a login`. + - *7.1.5.4.5.6.* Block sweep. The first + `<article>` is often a card, not the page. A landing page + can hold its text in plain `<div>` and `<section>` elements + and use `<article>` only for testimonials. So the converter + reads every block of `<body>`, scores each one with fixed + rules, and writes it as `keep`, `review`, or `drop`. The + script decides what is worth taking. The agent decides about + the `review` blocks when it refines the page. The rules name + no site. They read only HTML structure, so the same rules + serve a landing page, an encyclopedia article, a Q&A page, + and a forum thread. + - *7.1.5.4.5.6.1.* The sweep walks `<body>` in document order, + after the drops in *7.1.5.4.5.2.* A `template` with a + `shadowrootmode` attribute is a declarative shadow root. It + is not dropped, and the sweep reads its content in place. + A block is the outermost `h1` to `h6`, `p`, `ul`, `ol`, + `dl`, `blockquote`, `pre`, `table`, or `figure`. A `table` + whose cells hold another `table`, a `div`, or any block + element is a layout table. It is not a block, and the sweep + reads its cells as containers. + - *7.1.5.4.5.6.1.1.* An inline element is `a`, `abbr`, `b`, + `bdi`, `br`, `cite`, `code`, `data`, `del`, `dfn`, `em`, + `font`, `i`, `img`, `ins`, `kbd`, `label`, `mark`, `q`, `s`, + `samp`, `small`, `span`, `strong`, `sub`, `sup`, `time`, + `u`, `var`, or `wbr`. Every other element is a container. + That includes a custom element, whose name holds a hyphen, + such as `<my-comment>`. + - *7.1.5.4.5.6.1.2.* A run of text nodes and inline elements + that sits directly in a container is one loose-text block. + So `<div><span>Fast</span> evals</div>` is one block with + the text `Fast evals`, and so is `<my-post>Fast evals</my-post>`. + - *7.1.5.4.5.6.2.* The text of a block is its text with + whitespace runs folded to one space and trimmed. The link + text is the part of that text inside `<a>`. The word count + splits the text on spaces. + - *7.1.5.4.5.6.3.* A block is `drop` when any of these holds: + - its text has no letter or digit; + - it has fewer words than `fiona.wiki.ingest.html.min_words` + and its case-folded text equals that of an earlier `keep` + or `review` block, such as a repeated `Share` or `Reply`; + - it sits inside `nav`, `button`, `select`, an element with + the `hidden` attribute, an element with + `aria-hidden="true"`, or an element whose `style` + attribute sets `display: none` or `visibility: hidden`. + - *7.1.5.4.5.6.4.* A block that is not `drop` is `review`, with + the first reason that holds, in this order: + - `chrome`: it sits inside `aside`, `form`, `dialog`, an + element whose `role` is `navigation`, `banner`, + `contentinfo`, or `complementary`, or a `header` or + `footer` that is not inside `<article>` or `<main>`. + - `repeat`: its case-folded text equals that of an earlier + `keep` or `review` block. A long repeat can be a quote in + a reply or the same code in two answers, so the agent + decides. + - `links`: link text over text length is greater than + `fiona.wiki.ingest.html.max_link_density`. + - `short`: a `p` or loose-text block has fewer words than + `fiona.wiki.ingest.html.min_words`. Headings, lists, + `pre`, and tables are never `short`. + - *7.1.5.4.5.6.5.* Every other block is `keep`. + - *7.1.5.4.5.6.6.* The body is each `keep` and `review` block + in document order, each converted with the options of + *7.1.5.4.5.4.*, separated by one blank line. A `review` + block sits between the line `<!-- fiona:review <reason> -->` + and the line `<!-- /fiona:review -->`. A `drop` block is not + written. + - *7.1.5.4.5.6.7.* The warnings gain one entry, + `html sweep: <k> keep, <r> review, <d> drop`. When no block + is `keep` and at least one is `review`, the result is + `partial` with the warning `no block passed the content + checks; every block is marked for review`. + - *7.1.5.4.5.6.8.* The same bytes and settings give the same + body. The sweep reads no clock, network, or random source. + - *7.1.5.4.5.6.9.* When the agent refines the page, it keeps + each `review` block by removing its two marker lines, or + deletes the block with its markers. A refined page holds no + `fiona:review` marker. + - *7.1.5.4.5.6.10.* The sweep reads only the HTML it is given. + Text that JavaScript builds after load, comments behind a + `load more` control, and pages behind a login, consent wall, + or bot check are not in that HTML. Those cases are + *7.1.5.4.5.5.*, *7.1.5.2.8.*, and *7.1.5.14.*, not the + sweep. + - *acceptance*: A fixture with no `<main>`, a `<section>` + holding `<h2>How it works</h2>` and a `<div>` of twelve + words, and then two `<article>` testimonial cards, gives a + body holding the heading, the twelve words, and both card + quotes, in that order. + - *acceptance*: In a fixture with a nav list, a footer + paragraph, a `<span>•</span>`, a `<span>Share</span>` twice, + a twelve-word paragraph twice, a list whose text is all + links, and a two-word `<div>`, the nav text and the bullet + are absent, `Share` appears once, and the footer paragraph, + the second copy of the paragraph, the link list, and the + two-word text sit inside `fiona:review` markers with the + reasons `chrome`, `repeat`, `links`, and `short`. + - *acceptance*: An encyclopedia-style fixture with a side + menu in `nav`, an infobox table, body paragraphs with + `<sup>` citation links, a reference list, and a navbox + table of links keeps every body paragraph outside review + markers. The reference list and the navbox are `links`, and + no menu text is in the body. + - *acceptance*: A Q&A-style fixture with a question, two + answers that hold `pre` blocks, the same `pre` in both + answers, the vote counts `<div>42</div>`, `<div>7</div>`, + and `<div>3</div>` on the three posts, and a + related-questions `aside` keeps the question and both + answers in order. The second `pre` is `repeat`, each vote + count is `short`, and the aside is `chrome`. + - *acceptance*: A forum-style fixture whose posts are custom + elements, one post inside a + `<template shadowrootmode="open">`, nested replies, and a + hidden `style="display: none"` reply keeps the text of + every visible post and reply in document order, holds the + shadow-root post, and has no text of the hidden reply. + - *acceptance*: A layout table whose cells hold `div` posts + gives one block per post, not a pipe table. + - *acceptance*: Two conversions of the same fixture give + byte-identical bodies, and the warnings hold the + `html sweep:` entry with the counts of that fixture. + - *acceptance*: Fiona skill section 18 has the step in + *7.1.5.4.5.6.9.* - *acceptance*: For a fixture with a nav, an article, a script, a relative link `/a`, and a table, fetched from `https://host/p`, the body holds the article text, no script @@ -2873,7 +3146,9 @@ Candidates for later formats are in Appendix C. directory, and the page inside it is `<slug>.md`. After resolving `..` and symlinks the path must be inside `wiki/`, and it must not be `wiki/index.md`, `wiki/log.md`, `wiki/schema.md`, - or a file under `wiki/logs/`. Else exit 1. + or a file under `wiki/logs/`. Else exit 1. `NOT_IMPLEMENTED` The + page is then staged at `refine/` plus the same relative path + (*7.1.5.15.1.*). - *acceptance*: `-t concepts` writes `wiki/concepts/<slug>.md`. `-t ../raw/x.md` and `-t index.md` exit 1. @@ -2901,7 +3176,7 @@ Candidates for later formats are in Appendix C. sources: [<raw path>] source_meta: {<key>: <value>} ingest: - origin: <url, or resolved file path> + origin: <url, or file path (*7.1.5.2.10.*)> origin_kind: <url or file> final_url: <url after redirects, or null> fetched: <UTC time, YYYY-MM-DDTHH:MM:SSZ> @@ -3001,7 +3276,8 @@ Candidates for later formats are in Appendix C. `status` is `refined`, `version` is `1.0.0` or later, the quality signals are set and `llm.model` is not `none`, the page is listed in `wiki/index.md`, and `wiki/log.md` has a line for the - refinement. + refinement. `NOT_IMPLEMENTED` It is also under `wiki/`, not + `refine/` (*7.1.5.15.6.*). - *acceptance*: Fiona skill section 18 lists those conditions and the steps that meet them, and says when to convert a source by hand. @@ -3024,7 +3300,7 @@ Candidates for later formats are in Appendix C. and exit 4, stdout is empty. - *acceptance*: The stdout of an exit 3 run parses with `json.loads` and has `conversion: none`. -- *7.1.5.14.* `NOT_IMPLEMENTED` On exit 4 the agent gets the source +- *7.1.5.14.* On exit 4 the agent gets the source another way, or asks the user for it. The Fiona skill teaches this in section 18.3. The steps, in order: - *7.1.5.14.1.* Use a tool the agent already has that can read @@ -3050,6 +3326,193 @@ Candidates for later formats are in Appendix C. - *acceptance*: Fiona skill section 6.7 lists exit 4. Section 18.3 has the three cases above (a tool with access, ask the user, never ask for credentials) and shows the `--origin` command. +- *7.1.5.15.* `NOT_IMPLEMENTED` Staging in `refine/`. The Write stage + puts the raw page under `refine/` at the wiki root, not under + `wiki/`. `refine/` is the staging area between the tool's phase and + the agent's. Nothing in it is part of the wiki yet. The agent + refines a page there, then moves it into `wiki/`. So `wiki/` holds + only refined pages, and an unfinished ingest stays visible in one + place. When built, this replaces the `wiki/` page path of + *7.1.5.5.1.*, the existing-page check of *7.1.5.5.2.*, the + `--pending` list of *7.1.5.9.*, and the housekeeping `ingest` step + of *7.2.4.*. + - *7.1.5.15.1.* `NOT_IMPLEMENTED` A staged page mirrors its place in + the wiki. The page for `wiki/<rel>` is staged at `refine/<rel>`. + `<rel>` comes from *7.1.5.5.1.* unchanged: + `<default_target>/<slug>.md`, or the `-t/--target` value. That + value still names the page's final place under `wiki/` and is + checked against `wiki/` as before. Both paths sit at the same + depth, so a relative link resolves to the same target from + either. + - *acceptance*: A default ingest writes + `refine/sources/<slug>.md`, and its link + `../../raw/articles/<slug>.html` resolves. `-t concepts` writes + `refine/concepts/<slug>.md`. `-t ../raw/x.md` and `-t index.md` + still exit 1. Under `wiki/`, only `wiki/log.md` changes. + - *7.1.5.15.2.* `NOT_IMPLEMENTED` Reports name the staged path. The + `Wrote:` line, the JSON `page_path`, and the log bullet of + *7.1.5.6.* use `refine/<rel>`. The last stdout line is + `Next: refine refine/<rel>, then move it to wiki/<rel>.` + - *acceptance*: `--json` `page_path` starts with `refine/`, and + the log bullet ends `-> refine/<rel> (raw)`. `--dry-run` + prints the same path and writes nothing. + - *7.1.5.15.3.* `NOT_IMPLEMENTED` An existing page at `wiki/<rel>` + or `refine/<rel>` exits 1 unless `--force` is set, with the + *7.1.5.5.2.* message naming the page found, `wiki/<rel>` first. + With `--force`, the run writes `refine/<rel>` and never writes or + deletes `wiki/<rel>`. A refined page changes only when the agent + merges the staged page into it (*7.1.5.15.6.*). + - *acceptance*: With a refined `wiki/sources/a.md`, a second + ingest of its source exits 1 and names `wiki/sources/a.md`. + With `--force` it writes `refine/sources/a.md`, and the bytes + of `wiki/sources/a.md` are unchanged. + - *7.1.5.15.4.* `NOT_IMPLEMENTED` The duplicate check of + *7.1.5.3.5.* looks at pages under `refine/` and under `wiki/`. + `duplicate_page` is the path where the match was found. + - *acceptance*: A second ingest of the same bytes while the first + page is still staged reports + `duplicate_page: refine/sources/<slug>.md` and writes nothing. + - *7.1.5.15.5.* `NOT_IMPLEMENTED` `--pending` lists every markdown + page under `refine/` except `refine/README.md`, whatever its + `status`. Then it lists every page under `wiki/` with + `status: raw`, left there by an older version. Each group is + sorted by path. + - *acceptance*: One staged page and one `status: raw` page under + `wiki/` give `Raw pages: 2`, the staged page first. + - *7.1.5.15.6.* `NOT_IMPLEMENTED` Promotion is the agent's step. The + tool never moves a page out of `refine/`. *7.1.5.10.* adds one + condition: a refined page is under `wiki/`. The agent refines + `refine/<rel>` to every other condition of *7.1.5.10.*, moves it + to `wiki/<rel>`, and deletes any directory under `refine/` that + the move left empty. When `wiki/<rel>` already exists, as after a + `--force` re-ingest, the agent merges the staged page into it + instead. The merge keeps `created`, bumps `version`, adds the new + raw path to `sources`, and then deletes the staged page. + - *acceptance*: Fiona skill section 18 has the move step and the + merge step, and says a refined page is never under `refine/`. + - *7.1.5.15.7.* `NOT_IMPLEMENTED` A staged page writes each link as + it resolves from `wiki/<rel>`, so `../concepts/x.md` means + `wiki/concepts/x.md`. `wiki sanity` therefore skips `refine/` in + markdownlint and mdlinter. fmlint still checks it, so every staged + page keeps `name` and `version`. A page under `wiki/` that links + into `refine/` is a dead link (MDL001) as before. + - *acceptance*: A staged page with an MD013 hit and a link + `../concepts/x.md` that exists only as `wiki/concepts/x.md` + gives no markdownlint and no mdlinter violation. The same page + without `version` gives an fmlint violation. + - *7.1.5.15.8.* `NOT_IMPLEMENTED` `wiki graph` omits `refine/`, the + same way it omits a path in `fiona.wiki.graph.ignore_files` + (*7.1.4.20.*). `wiki search` already omits it, because its + default scope is `wiki/**/*.md` (*7.1.6.1.1.*). + - *acceptance*: A staged page is absent from the graph JSON + `nodes`, and it is not an orphan in housekeeping. + - *7.1.5.15.9.* `NOT_IMPLEMENTED` The housekeeping `ingest` step is + `WARN` when any page below exists, with one detail and one action + per page, and `PASS` otherwise: + - *7.1.5.15.9.1.* A page under `refine/` without + `status: refined`: detail `staged: refine/<rel>`, action + `Refine refine/<rel>, then move it to wiki/<rel>.` + - *7.1.5.15.9.2.* A page under `refine/` with `status: refined`: + detail `ready: refine/<rel>`, action + `Move refine/<rel> to wiki/<rel>.`, or + `Merge refine/<rel> into wiki/<rel>.` when that page exists. + - *7.1.5.15.9.3.* A page under `wiki/` with `status: raw`: detail + `raw in wiki: wiki/<rel>`, action + `Move wiki/<rel> to refine/<rel> and refine it there.` + - *acceptance*: One page of each kind gives `WARN` and three + details, each with its action. `refine/README.md` alone gives + `PASS`. + - *7.1.5.15.10.* `NOT_IMPLEMENTED` The template ships + `refine/README.md`, with frontmatter `name` and `version`, which + says what the directory is for. `refine/` is a core directory + (*3.2.*) and is not read-only. `wiki update` adds + `refine/README.md` when it is missing and never touches a staged + page. `refine/` is not in `.gitignore`, so `git save` keeps staged + work. The template `AGENTS.md` boundaries and the + `wiki/schema.md` directory tree name `refine/`. + - *acceptance*: After `wiki init`, `git ls-files` lists + `refine/README.md`. `wiki update` on a wiki without `refine/` + adds that file and nothing else under `refine/`. The template + `AGENTS.md` and `wiki/schema.md` both contain `refine/`. + +#### 7.1.6. wiki search + +> Status: Planned + +Implements section 4.7.7. A plain keyword search over the wiki, so +the agent and the user find the lines that mention a term without +reading whole pages. It reads files and writes nothing. + +- *7.1.6.1.* `NOT_IMPLEMENTED` Files searched. + - *7.1.6.1.1.* By default the search covers every `*.md` file + under `wiki/`. Each `--include GLOB` is relative to the wiki + root, may repeat, and replaces that default. So + `--include "raw/**/*.md" --include "wiki/**/*.md"` searches both. + - *7.1.6.1.2.* A file is read as UTF-8 with any BOM stripped. + Bytes that do not decode are replaced, and the file is still + searched. Frontmatter lines are searched like any other line. + - *7.1.6.1.3.* Files are visited sorted by their relative path, + and lines in file order. + - *acceptance*: With a match in `wiki/a.md` and in `raw/b.md`, + the default prints only `wiki/a.md`. With + `--include "**/*.md"` both print, `raw/b.md` first. +- *7.1.6.2.* `NOT_IMPLEMENTED` Matching. + - *7.1.6.2.1.* A keyword is a literal substring, not a regex. The + match ignores case unless `--case-sensitive` is set. + - *7.1.6.2.2.* A line matches when it holds any keyword. With + `--all-words`, it matches only when it holds every keyword. + - *7.1.6.2.3.* An empty keyword, or one that is only whitespace, + exits 1 with `Error: Empty keyword`. + - *acceptance*: For the line `Agent Harness works`, `harness` + matches, `harness --case-sensitive` does not, `harness eval` + matches, and `harness eval --all-words` does not. `a.b` does + not match `axb`. +- *7.1.6.3.* `NOT_IMPLEMENTED` Output. Each matching line prints + once, as `<path>:<line>: <text>`, even when it holds several + matches. The last line is `Matches: <N> lines in <F> files`. + - *acceptance*: Two matches on line 7 of `wiki/a.md` print one + line that starts with `wiki/a.md:7:` and one space, and the + summary is + `Matches: 1 lines in 1 files`. +- *7.1.6.4.* `NOT_IMPLEMENTED` Long lines. `<text>` is the line with + leading and trailing whitespace removed. When it is longer than + `fiona.wiki.search.max_line_chars`, it is shortened around the + first match on that line: + - *7.1.6.4.1.* The window is that many characters, minus room for + the markers below. It holds the whole first match and splits + the remaining room evenly before and after it. When one side + reaches the start or end of the line, the other side takes the + unused room. + - *7.1.6.4.2.* A window edge that falls inside a word moves to + the nearest space inside the window, unless that would cut the + match. + - *7.1.6.4.3.* The marker `...` and one space start the text when + the window does not start the line. One space and `...` end it + when the window does not end the line. The result, with + markers, is no longer than `max_line_chars`. + - *7.1.6.4.4.* When the match alone is longer than the window, + the window starts at the match and is cut at its end. + - *acceptance*: With `max_line_chars` 40 and a 200-character line + whose only match `harness` starts at column 100, `<text>` has + both markers, is at most 40 characters, and holds `harness` + whole. The text between the markers starts and ends on a whole + word. + - *acceptance*: With the match in the first 10 characters of that + line, `<text>` starts with the line's first word, has no + start marker, and has the end marker. + - *acceptance*: A line of 40 characters or fewer prints unchanged. +- *7.1.6.5.* `NOT_IMPLEMENTED` Limit. Only the first + `fiona.wiki.search.max_results` match lines print. When more + exist, the line `... <N> more matches. Narrow the keywords or + raise max_results.` follows them. The summary and the JSON + `total` count every match, and JSON `truncated` is `true`. + - *acceptance*: With `max_results` 2 and five matching lines, two + match lines print, then `... 3 more matches. ...`, then + `Matches: 5 lines in <F> files`. +- *7.1.6.6.* `NOT_IMPLEMENTED` The command writes no file and no + `wiki/log.md` line. + - *acceptance*: Every file under the wiki has the same bytes and + modification time after a search. ### 7.2. housekeeping @@ -3101,6 +3564,8 @@ Paths in details and actions are relative to the wiki root. frontmatter has `status: raw`. It is `WARN` when one exists, with one detail per page and the action `Refine each raw page, then set status: refined.` + `NOT_IMPLEMENTED` *7.1.5.15.9.* replaces this step once staging is + built. - *acceptance*: A page with `status: raw` is `WARN` and named in the details. With none, the step is `PASS`. - *7.2.5.* `sanity` runs the checks of *7.1.3.* without the progress @@ -3155,7 +3620,14 @@ Requirements for `librarian`. or `v0.0.0` when version is missing or unreadable. - *7.3.1.1.3.* Headers become nested GitHub-style anchor links. Fenced code is not scanned. Files are read as `utf-8-sig` so a - BOM does not hide the first header. + BOM does not hide the first header. The anchor is the header + text lowercased, with every character that is not a word + character, space, or `-` removed, and then each space replaced by + one `-`. Runs of spaces are not collapsed, so the anchor matches + the one PyMarkdown MD051 and Markdown previews resolve. + - *acceptance*: `Training | XXXXXXX-x-xxxx DONE` has the anchor + `training--xxxxxxx-x-xxxx-done`, and `timeout / cannot` has + `timeout--cannot`. - *7.3.1.1.4.* Skip the TOC file itself, hidden folders, and `--exclude` matches. - *7.3.1.1.5.* `--recursive-toc` writes a TOC in every subfolder. @@ -3886,7 +4358,10 @@ is the pip-argument source both scripts call. | P2 | settings setup | 7.16 | Planned | | P2 | Bootstrap install modes | 7.21 | Implemented | | P2 | wiki ingest | 7.1.5 | Partially Implemented | -| P2 | Ingest login detection and `--origin` | 7.1.5.2.8, 7.1.5.2.9, 7.1.5.14 | Planned | +| P2 | Ingest login detection and `--origin` | 7.1.5.2.8, 7.1.5.2.9, 7.1.5.14 | Implemented | +| P2 | HTML block sweep | 7.1.5.4.5.6 | Implemented | +| P2 | wiki search | 4.7.7, 7.1.6 | Planned | +| P2 | Staging in `refine/` | 7.1.5.15 | Planned | | P3 | `--version` | 2.3 | Planned | | P3 | fmlint enforcement of quality signals | 6.2 | Planned | | P3 | Backup archive | 7.13 | Planned | @@ -3948,6 +4423,17 @@ is the pip-argument source both scripts call. - On exit 4, `--json` prints nothing (*7.1.5.13.*). A JSON error object such as `{"error": "auth_required", "url": ..., "reason": ...}` would save the agent parsing stderr. Not decided. +- The block sweep (*7.1.5.4.5.6.3.*) drops a block whose text has no + letter or digit, so an image or figure with no caption text is + not in the page. Counting `alt` text, or keeping such a block as + `review`, is not decided. +- Per-host URL rewrites before fetch (for example Reddit to + `old.reddit.com`), and pages that build their text with + JavaScript, are not specified. +- Moving a page from `refine/` to `wiki/` is a hand step for the agent + (*7.1.5.15.6.*). A `wiki promote <page>` command could check the + conditions of *7.1.5.10.*, move or merge the page, and append the + log line. Not decided. --- @@ -3983,11 +4469,13 @@ Layout for section 7.1.5, under `src/fiona/wiki/ingest/`: | Module | Classes | Role | | --- | --- | --- | -| `models.py` | `FetchedSource`, `ConversionResult`, `IngestResult` | Dataclasses passed between stages. `IngestResult.to_dict()` is the JSON in section 4.7.6. | -| `fetchers.py` | `SourceFetcher` (ABC), `LocalFileFetcher`, `UrlFetcher` | `accepts(argument)` and `fetch(argument, dest_dir)`. `UrlFetcher` takes the opener. | +| `models.py` | `FetchedSource`, `ConversionResult`, `IngestResult`, `IngestError`, `AuthRequiredError` | Dataclasses passed between stages. `IngestResult.to_dict()` is the JSON in section 4.7.6. `AuthRequiredError` is exit 4. | +| `fetchers.py` | `SourceFetcher` (ABC), `LocalFileFetcher`, `UrlFetcher` | `accepts(argument)` and `fetch(argument, dest_dir)`. `UrlFetcher` takes the opener and runs the login checks of *7.1.5.2.8.2.* before writing. | +| `settings.py` | `IngestSettings`, `load_ingest_settings()` | `fiona.wiki.ingest` keys of section 5.1. | | `media.py` | `detect_media_type`, `make_slug` | *7.1.5.2.4.* and *7.1.5.2.6.* | | `converters/base.py` | `SourceConverter` (ABC), `ConverterRegistry`, `ConversionError` | Class attributes `name`, `media_types`, `extensions`, `raw_category`; `convert(source)`. `ConverterRegistry.find(media_type, extension)`. | -| `converters/markdown.py`, `text.py`, `html.py`, `pdf.py` | `MarkdownConverter`, `TextConverter`, `HtmlConverter`, `PdfConverter` | One format each. | +| `converters/markdown.py`, `text.py`, `html.py`, `pdf.py` | `MarkdownConverter`, `TextConverter`, `HtmlConverter`, `PdfConverter` | One format each. `HtmlConverter(min_words, max_link_density)`; `default_registry(settings)` passes the `html.*` settings. | +| `converters/html_sweep.py` | `Block`, `SweepResult`, `sweep()` | *7.1.5.4.5.6.* walk, classify, and render. | | `archiver.py` | `RawArchiver` | *7.1.5.3.*; `find_by_sha256(digest, size)` scans `raw/` for a reusable copy. | | `page_writer.py` | `RawPageWriter` | *7.1.5.5.* | | `log_writer.py` | `IngestLog` | *7.1.5.6.* | diff --git a/pyproject.toml b/pyproject.toml index 4ae9c76..692566b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "fiona-assistant" -version = "1.9.1" +version = "1.16.0" description = "Tools for managing the LLM wiki." authors = [{name = "nvaldeziii"}] readme = "README.md" diff --git a/src/fiona/data/default.gitignore b/src/fiona/data/default.gitignore index 1d3084a..9c4f262 100644 --- a/src/fiona/data/default.gitignore +++ b/src/fiona/data/default.gitignore @@ -47,6 +47,10 @@ htmlcov/ # Windows Thumbs.db +# Obsidian per-user vault state +.obsidian/workspace*.json +.obsidian/cache + # Python *.pyc *.pyo diff --git a/src/fiona/data/llm_wiki/AGENTS.md b/src/fiona/data/llm_wiki/AGENTS.md index 82ba538..cc69933 100644 --- a/src/fiona/data/llm_wiki/AGENTS.md +++ b/src/fiona/data/llm_wiki/AGENTS.md @@ -1,9 +1,9 @@ --- name: llm-wiki-agent-rules title: LLM Wiki Agent Rules -version: 1.3.0 +version: 1.4.0 created: 2025-09-23 -updated: 2026-09-30 +updated: 2026-10-14 type: summary --- @@ -33,6 +33,14 @@ rest of the session. It loads llm-wiki-mod. When that skill is already loaded, do not read it again. This file does not repeat it. +Then read [MEMORY](./MEMORY.md): what this wiki's owner wants, kept +across sessions. + +Update `MEMORY.md` whenever an important memory comes up: a lasting +preference, a correction, a repeated request, or a fact a run +verified. Also update it whenever the user explicitly asks you to +remember something. Section 1 of that file says how. + ## 2. What an llm-wiki is A directory is an llm-wiki when it contains `fiona-settings.json`. diff --git a/src/fiona/data/llm_wiki/MEMORY.md b/src/fiona/data/llm_wiki/MEMORY.md new file mode 100644 index 0000000..6588463 --- /dev/null +++ b/src/fiona/data/llm_wiki/MEMORY.md @@ -0,0 +1,179 @@ +--- +name: memory +version: 1.0.0 +title: Agent Memory +created: DATE_PLACEHOLDER +updated: DATE_PLACEHOLDER +type: summary +--- + +# Agent Memory + +<!-- toc-start: headings 2..6 --> + +- [1. How to use this file](#1-how-to-use-this-file) + - [1.1 When to read](#11-when-to-read) + - [1.2 When to write](#12-when-to-write) + - [1.3 How to write](#13-how-to-write) + - [1.4 How the memory evolves](#14-how-the-memory-evolves) +- [2. Pages and writing](#2-pages-and-writing) + - [2.1 Style and wording](#21-style-and-wording) + - [2.2 Placement and links](#22-placement-and-links) +- [3. Sources and ingest](#3-sources-and-ingest) + - [3.1 Hosts and access](#31-hosts-and-access) + - [3.2 Conversion and refinement](#32-conversion-and-refinement) +- [4. Tools and environment](#4-tools-and-environment) + - [4.1 Commands and flags](#41-commands-and-flags) + - [4.2 Machine and shell](#42-machine-and-shell) +- [5. History and housekeeping](#5-history-and-housekeeping) + - [5.1 Saves and maintenance](#51-saves-and-maintenance) + +<!-- toc-end --> + +What this wiki's owner wants from you, kept across sessions: +standing preferences, corrections, repeated request patterns, and +facts a run verified about operating this wiki. Knowledge from +sources does not go here; it goes to pages under `wiki/`. + +## 1. How to use this file + +### 1.1 When to read + +- **1.1.1** Read this file once the + [Fiona skill](./utils/skills/fiona/SKILL.md) is loaded, before the + first wiki work of the session. When it is already in context, do + not read it again. +- **1.1.2** Apply every entry that matches the request. + - **1.1.2.1** An entry overrides a default of the Fiona skill or + the [schema](./wiki/schema.md), for this wiki only. + - **1.1.2.2** An entry never overrides the boundaries in section 3 + of [AGENTS](./AGENTS.md); when the two conflict, follow the + boundary and ask the user. + - **1.1.2.3** An entry never changes a tool contract: a command, a + flag, or a required frontmatter field. +- **1.1.3** Every heading and list item is numbered as + [sectionize-markdown](./utils/skills/sectionize-markdown/sectionize-markdown.md) + describes. Cite an item as `MEMORY.md item 2.1.3`; a new item takes + the next free number in its subsection. + +### 1.2 When to write + +- **1.2.1** Write an entry when one of these happens: + - **1.2.1.1** The user states a requirement or preference that + outlasts the current request ("always", "never", "from now on", + "in this wiki", "we use"). + - **1.2.1.2** The user corrects your output or approach: a page, a + summary, a placement, a link, or a command you ran. + - **1.2.1.3** The same kind of request comes for the second time; + record the pattern and what the user wanted. + - **1.2.1.4** A run verifies a reusable fact about operating this + wiki that the Fiona skill and the schema do not hold, such as a + host that needs a login, a converter that loses a site's tables, + or a flag combination that works. + - **1.2.1.5** The user explicitly asks you to remember something + ("remember", "note this", "save to memory"). +- **1.2.2** Do not write: + - **1.2.2.1** One-off values that only apply to the current + request. + - **1.2.2.2** Passwords, tokens, cookies, personal data, or + anything the user asks to keep out. + - **1.2.2.3** Knowledge from a source. That is a page under + `wiki/`, written by an ingest. + - **1.2.2.4** Facts that the Fiona skill, the schema, or a wiki + page already states; cite them instead. + - **1.2.2.5** Guesses; only what the user said or a run showed. + +### 1.3 How to write + +- **1.3.1** Put the entry under the `###` subsection that fits. + - **1.3.1.1** When none fits, add a `###` subsection with the next + free number under the matching `##`, or a new `##` category with + the next free number. + - **1.3.1.2** Never put an entry directly under a `##`; its id + would collide with a later subsection. +- **1.3.2** Use the knowledge-entry shape of sectionize-markdown + section 4: a header line with the source tag and a short title, + one child per fact, and a last child with the ref and the date. +- **1.3.3** Source tags: `user` (the user said it), `run` (a command + or tool run showed it), `source` (a file under `raw/` states it), + joined with `+` when several agree. +- **1.3.4** The ref says where to check the fact: the user's request + and its date, the command and its output line, or a page or raw + path. +- **1.3.5** Write the entry in the same turn, then tell the user + which item was added or changed, for example "Saved to MEMORY.md + item 2.1.1". +- **1.3.6** Bump `version` (patch for a changed entry, minor for a + new entry or subsection) and set `updated` in the frontmatter on + every change. +- **1.3.7** Append `- YYYY-MM-DD - memory: <item id> <title>` to + `wiki/log.md`, and bump that file too. +- **1.3.8** The change lands in the save of the current ingest cycle + or housekeeping pass (Fiona skill section 2.3). + - **1.3.8.1** With no cycle running, save it on its own with + `fiona-assistant git save --description "memory: <item id> <title>"`. + +An entry looks like this: + +```text +- **2.1.1** [user] "Source summaries open with the decision" + - **2.1.1.1** A source page starts with the decision the source records, then the reasons. + - **2.1.1.2** Applies to meeting notes and design reviews. + - **2.1.1.3** Ref: user request YYYY-MM-DD; added YYYY-MM-DD. +``` + +### 1.4 How the memory evolves + +- **1.4.1** Before adding an entry, look for a near-duplicate in the + same category; extend it instead and keep its id and its earliest + date. +- **1.4.2** A finding that contradicts an entry becomes its next + child, titled `OVERRIDE: ...`, with the new facts and its own ref. +- **1.4.3** An entry that no longer applies is marked + **Superseded YYYY-MM-DD** with the reason; it is not deleted. +- **1.4.4** An entry that held for several uses graduates; mark it + **Promoted YYYY-MM-DD** with the target. + - **1.4.4.1** A convention for every page moves to the + [schema](./wiki/schema.md). + - **1.4.4.2** A reusable procedure moves to a skill under + [utils/skills_local](./utils/skills_local/README.md). +- **1.4.5** Never renumber ids; pages, log lines, and chats may cite + them. +- **1.4.6** This file belongs to this wiki and evolves with it. + - **1.4.6.1** `fiona-assistant wiki update` adds it only when it is + missing. It never compares or replaces it, and never writes a + `MEMORY-update.md`. + - **1.4.6.2** A newer template version of these rules does not + reach an existing wiki; the owner decides what to adopt. +- **1.4.7** Git is the memory's history: `git log -p MEMORY.md` shows + when each entry changed and why. + +## 2. Pages and writing + +How pages read, where they go, and how they link. + +### 2.1 Style and wording + +### 2.2 Placement and links + +## 3. Sources and ingest + +Where sources come from and how they become pages. + +### 3.1 Hosts and access + +### 3.2 Conversion and refinement + +## 4. Tools and environment + +How `fiona-assistant` and the machine behave for this wiki. + +### 4.1 Commands and flags + +### 4.2 Machine and shell + +## 5. History and housekeeping + +Saves, commits, merges, and maintenance passes. + +### 5.1 Saves and maintenance diff --git a/src/fiona/data/llm_wiki/constellation/colorful.json b/src/fiona/data/llm_wiki/constellation/colorful.json index f03edea..d8ee318 100644 --- a/src/fiona/data/llm_wiki/constellation/colorful.json +++ b/src/fiona/data/llm_wiki/constellation/colorful.json @@ -1,5 +1,5 @@ { - "version": "1.0.0", + "version": "1.1.0", "background": "#f4efe6", "graph_background": "#fffdf8", "font_family": "Arial, sans-serif", @@ -28,6 +28,10 @@ "clicked": { "node_color": "#0e6b66" }, + "orphan": { + "enabled": false, + "node_color": "#7b5ea7" + }, "buttons": { "background": "#0f6e78", "text": "#fffdf8", diff --git a/src/fiona/data/llm_wiki/constellation/dark.json b/src/fiona/data/llm_wiki/constellation/dark.json index 9d42e29..bb248cb 100644 --- a/src/fiona/data/llm_wiki/constellation/dark.json +++ b/src/fiona/data/llm_wiki/constellation/dark.json @@ -1,5 +1,5 @@ { - "version": "1.0.0", + "version": "1.1.0", "background": "#1e2836", "graph_background": "#283448", "font_family": "Segoe UI, sans-serif", @@ -28,6 +28,10 @@ "clicked": { "node_color": "#ffe3a3" }, + "orphan": { + "enabled": false, + "node_color": "#ef8a7a" + }, "buttons": { "background": "#3e5674", "text": "#f4f8fc", diff --git a/src/fiona/data/llm_wiki/constellation/grayscale.json b/src/fiona/data/llm_wiki/constellation/grayscale.json index c17ceed..691597d 100644 --- a/src/fiona/data/llm_wiki/constellation/grayscale.json +++ b/src/fiona/data/llm_wiki/constellation/grayscale.json @@ -1,5 +1,5 @@ { - "version": "1.0.0", + "version": "1.1.0", "background": "#e6e6e6", "graph_background": "#f4f4f4", "font_family": "Arial, sans-serif", @@ -28,6 +28,10 @@ "clicked": { "node_color": "#000000" }, + "orphan": { + "enabled": false, + "node_color": "#bdbdbd" + }, "buttons": { "background": "#2a2a2a", "text": "#ffffff", diff --git a/src/fiona/data/llm_wiki/constellation/night-sky.json b/src/fiona/data/llm_wiki/constellation/night-sky.json index 78f2df2..b360fc2 100644 --- a/src/fiona/data/llm_wiki/constellation/night-sky.json +++ b/src/fiona/data/llm_wiki/constellation/night-sky.json @@ -1,5 +1,5 @@ { - "version": "1.0.0", + "version": "1.1.0", "background": "#070b14", "graph_background": "#0b1020", "font_family": "Segoe UI, sans-serif", @@ -28,6 +28,10 @@ "clicked": { "node_color": "#ffd56a" }, + "orphan": { + "enabled": false, + "node_color": "#ff7f6e" + }, "buttons": { "background": "#1e2a4a", "text": "#e8eeff", diff --git a/src/fiona/data/llm_wiki/constellation/professional.json b/src/fiona/data/llm_wiki/constellation/professional.json index a6c86e4..def0c82 100644 --- a/src/fiona/data/llm_wiki/constellation/professional.json +++ b/src/fiona/data/llm_wiki/constellation/professional.json @@ -1,5 +1,5 @@ { - "version": "1.0.0", + "version": "1.1.0", "background": "#f7fafc", "graph_background": "#fcfdfe", "font_family": "Segoe UI, Calibri, Helvetica Neue, Arial, sans-serif", @@ -28,6 +28,10 @@ "clicked": { "node_color": "#5b93b0" }, + "orphan": { + "enabled": false, + "node_color": "#d9a5b3" + }, "buttons": { "background": "#8eb4c9", "text": "#ffffff", diff --git a/src/fiona/data/llm_wiki/fiona-settings.json b/src/fiona/data/llm_wiki/fiona-settings.json index 2cec50e..847eabc 100644 --- a/src/fiona/data/llm_wiki/fiona-settings.json +++ b/src/fiona/data/llm_wiki/fiona-settings.json @@ -1,5 +1,5 @@ { - "version": "1.0.0", + "version": "1.0.1", "markdown": { "pymarkdown": { "extensions": [ @@ -76,7 +76,9 @@ "consolidate_external_links_depth": 3, "consolidate_external_links_max_char": 8, "ignore_files": [ - "*/table_of_contents.md" + "*/table_of_contents.md", + "*/README.md", + "*/AGENTS.md" ] }, "update": { diff --git a/src/fiona/data/llm_wiki/utils/skills/README.md b/src/fiona/data/llm_wiki/utils/skills/README.md index 3766ffe..b650c80 100644 --- a/src/fiona/data/llm_wiki/utils/skills/README.md +++ b/src/fiona/data/llm_wiki/utils/skills/README.md @@ -1,12 +1,18 @@ --- name: skills -version: 1.0.0 +version: 1.1.0 title: Skills Directory type: summary created: 2025-09-23 -updated: 2026-09-24 +updated: 2026-10-15 --- # Skills Directory Skill definitions and configurations. + +- [fiona](./fiona/SKILL.md): operate the llm-wiki with `fiona-assistant`. +- [memory](./memory/SKILL.md): give a skill its own `MEMORIES.md`. +- [python-tdd](./python-tdd/SKILL.md): test-driven Python work. +- [sectionize-markdown](./sectionize-markdown/sectionize-markdown.md): + number headings and list items so each rule can be cited. diff --git a/src/fiona/data/llm_wiki/utils/skills/fiona/SKILL.md b/src/fiona/data/llm_wiki/utils/skills/fiona/SKILL.md index 805d0f4..0a2a1e7 100644 --- a/src/fiona/data/llm_wiki/utils/skills/fiona/SKILL.md +++ b/src/fiona/data/llm_wiki/utils/skills/fiona/SKILL.md @@ -3,8 +3,8 @@ name: fiona category: productivity description: Operate an llm-wiki with the fiona-assistant CLI. author: nvaldez -version: 2.0.0 -updated: 2026-10-06 +version: 2.2.0 +updated: 2026-10-14 --- # Fiona @@ -365,6 +365,10 @@ Update complete: <U> file(s) updated, <A> added, <C> conflict(s), <S> skipped. Merge each <stem>-update<suffix> into its file: run fiona-assistant housekeeping. ``` +**6.4.13** `MEMORY.md` belongs to the wiki. Update adds it only when +it is missing; it never compares or replaces it, and never writes a +`MEMORY-update.md`. + ### 6.5. sanity ```text @@ -409,7 +413,7 @@ contents links to every page and would hide the real links. ### 6.7. ingest ```text -fiona-assistant wiki ingest <source> [--wiki DIR] [-r|--raw PATH] [-t|--target PATH] [--raw-only] [-f|--force] [--dry-run] [--json] +fiona-assistant wiki ingest <source> [--wiki DIR] [-r|--raw PATH] [-t|--target PATH] [--origin URL] [--raw-only] [-f|--force] [--dry-run] [--json] fiona-assistant wiki ingest --pending [--wiki DIR] [--json] ``` @@ -417,7 +421,8 @@ fiona-assistant wiki ingest --pending [--wiki DIR] [--json] path. The tool is the first phase of an ingest: - **6.7.1.1** A URL is downloaded into `ingest/` as whatever it is: - HTML, PDF, text. No credentials are sent. + HTML, PDF, text. No credentials are sent. A URL that needs a login + stops with exit 4 and writes nothing (section 18.3.4). - **6.7.1.2** The file is archived under `raw/`. A file in `ingest/` is moved. Any other file is copied. Nothing already in `raw/` is changed. @@ -439,6 +444,7 @@ touch any other page. That is your phase, section 18. | --- | --- | | `-r`, `--raw PATH` | Place the raw copy, relative to `raw/`. A directory (`-r jenkins/tl153`) or a file name with an extension. | | `-t`, `--target PATH` | Place the page, relative to `wiki/`. A directory (`-t concepts`) or a `.md` file name. | +| `--origin URL` | The `http(s)` URL a local file was saved from. Sets `ingest.origin` and `origin_kind: url`, and resolves relative links against it. Local files only. | | `--raw-only` | Archive only. Use it when you will convert by hand. | | `-f`, `--force` | Replace an existing page. Ask the user first when that page is refined. | | `--dry-run` | Print the plan. Writes nothing in the wiki. A URL is still downloaded, to a temporary directory. | @@ -457,7 +463,7 @@ names a place or the source belongs beside existing pages. | --- | --- | --- | | `markdown` | `.md` | Nothing. Source frontmatter moves to `source_meta`. | | `text` | `.txt`, `.log` | Structure. The body is one fenced `text` block. | -| `html` | web pages | Navigation, scripts, and anything drawn by JavaScript. | +| `html` | web pages | Navigation, buttons, hidden text, scripts, and anything drawn by JavaScript. Doubtful blocks stay, marked for review (section 18.5.2.1). | | `pdf` | PDFs with a text layer | Tables, images, columns, and scanned pages. | **6.7.7** Pass `--json` and read the object: `raw_path`, @@ -472,6 +478,20 @@ page frontmatter carries the same facts in its `ingest` map. | 0 | Page written, `--raw-only`, or a duplicate | Read `duplicate_page`, `conversion`, and `warnings`, then section 18.2. | | 1 | Error on stderr, nothing written | Fix the argument, or fetch the source another way (section 18.3). | | 3 | Raw copy archived, no page | Convert by hand (section 18.3). | +| 4 | The URL needs a login. Nothing written, stdout empty even with `--json` | Get the source another way (section 18.3.4). | + +**6.7.9** Exit 4 prints two lines on stderr. The reason is +`401 Unauthorized`, `403 Forbidden`, `redirected to <url>`, or +`login page at <url>`: + +```text +Error: Authentication required: <url>: <reason> +Next: save it into ingest/, then run: wiki ingest <file> --origin <url> +``` + +**6.7.10** A redirect counts as a login only when the final host and +path match `fiona.wiki.ingest.login_url_patterns`. A page whose own +URL holds `login`, with no redirect, still ingests. ## 7. librarian @@ -854,6 +874,9 @@ are done. The pattern is section 3.1 of - **18.2.4** On exit 0 with `conversion: complete` and no warning that lost content, check the metadata (section 18.4), then refine (section 18.5). Otherwise decide with section 18.3. + - **18.2.4.1** `html sweep: <k> keep, <r> review, <d> drop` only + counts the blocks of an HTML page. It is not lost content. +- **18.2.5** On exit 4, follow section 18.3.4. Do not retry the URL. ### 18.3. Convert by hand @@ -869,9 +892,8 @@ by hand when: - **18.3.1.3** Layout carries the meaning: tables, multi-column papers, forms, slides, spreadsheets, diagrams, or a bundle of logs. -- **18.3.1.4** The download failed on authentication (401, 403, or a - login page in the body). Fetch the source with a tool that has - access, save it into `ingest/`, and ingest that file. +- **18.3.1.4** The command exited 4: the URL needs a login. Section + 18.3.4 gets the file. The tool still converts it. **18.3.2** How: @@ -897,6 +919,31 @@ by hand when: **18.3.3** When the tool archived the wrong file or to the wrong place, tell the user. Do not move or delete anything under `raw/`. +**18.3.4** A URL that needs a login (exit 4). The tool cannot log +in. Get the source with these steps, in order: + +- **18.3.4.1** Use a tool you already have that reads that host with + the user's access: an MCP server for Confluence, SharePoint, + Jenkins, or a ticket system, or an authenticated script in + `utils/scripts/`. Save what it returns into `ingest/<slug>.<ext>` + in its own format: HTML as `.html`, a PDF as `.pdf`, plain text as + `.txt` or `.md`. Then run the `Next:` command with that file: + + ```text + fiona-assistant wiki ingest ingest/<slug>.<ext> --origin <url> --json + ``` + +- **18.3.4.2** When no tool has access, stop and ask the user to save + the page into `ingest/`, for example with the browser's Save As or + Print to PDF. Name the URL, the folder, and the command you will + run once the file is there. Do not retry the URL. +- **18.3.4.3** Never ask the user for a password, token, or cookie. + Never put credentials in a URL, a command line, a page, or the + log. +- **18.3.4.4** A tool that returns a summary instead of the source is + not the source. Archive what it returned as it is, and add a line + to `ingest.warnings` that names the tool it came through. + ### 18.4. Check the metadata **18.4.1** The tool fills every key in section 18.6, but it reads @@ -945,6 +992,20 @@ the metadata. - **18.5.2** Read the page and the source it cites. Open the raw file when `ingest.conversion` is `partial` or a claim looks garbled. + - **18.5.2.1** An HTML page can hold blocks between + `<!-- fiona:review <reason> -->` and `<!-- /fiona:review -->`. + The tool kept them but doubts them. Decide each one: keep it by + deleting its two marker lines, or delete the block with its + markers. A refined page holds no `fiona:review` marker. + - **18.5.2.2** `chrome`: it sits in an aside, form, dialog, landmark + role, or a page header or footer. Usually delete. + - **18.5.2.3** `repeat`: the same text appeared earlier. Keep it + only when the repeat matters, such as a quote in a reply. + - **18.5.2.4** `links`: mostly link text, such as a reference list + or a link menu. Keep references; delete menus. + - **18.5.2.5** `short`: a paragraph of fewer than `html.min_words` + words, such as a button label or a counter. Keep a number that + a nearby sentence explains. - **18.5.3** Rewrite the page as a source summary: - **18.5.3.1** The first paragraph says what the source is and why it matters. diff --git a/src/fiona/data/llm_wiki/utils/skills/memory/MEMORIES.md b/src/fiona/data/llm_wiki/utils/skills/memory/MEMORIES.md new file mode 100644 index 0000000..2fa978b --- /dev/null +++ b/src/fiona/data/llm_wiki/utils/skills/memory/MEMORIES.md @@ -0,0 +1,173 @@ +--- +name: skill-memories +version: 1.0.0 +title: SKILL_NAME Memories +created: DATE_PLACEHOLDER +updated: DATE_PLACEHOLDER +type: summary +--- + +# SKILL_NAME Memories + +<!-- toc-start: headings 2..6 --> + +- [1. How to use this file](#1-how-to-use-this-file) + - [1.1 When to read](#11-when-to-read) + - [1.2 When to write](#12-when-to-write) + - [1.3 How to write](#13-how-to-write) + - [1.4 How the memory evolves](#14-how-the-memory-evolves) +- [2. Output and wording](#2-output-and-wording) + - [2.1 Style and format](#21-style-and-format) + - [2.2 Placement and naming](#22-placement-and-naming) +- [3. Inputs and sources](#3-inputs-and-sources) + - [3.1 Where inputs come from](#31-where-inputs-come-from) + - [3.2 Quirks and parsing](#32-quirks-and-parsing) +- [4. Steps and tools](#4-steps-and-tools) + - [4.1 Commands and flags](#41-commands-and-flags) + - [4.2 Machine and shell](#42-machine-and-shell) +- [5. Recurring requests](#5-recurring-requests) + - [5.1 Patterns](#51-patterns) + +<!-- toc-end --> + +What users want from the SKILL_NAME skill, kept across sessions: +standing preferences, corrections, repeated request patterns, and +facts a run verified while using this skill. Preferences for the +whole wiki go to the wiki's `MEMORY.md`. Knowledge from sources goes +to pages under `wiki/`. + +## 1. How to use this file + +### 1.1 When to read + +- **1.1.1** Read this file once [the skill](./SKILL.md) is loaded, + before its first step of the session. When it is already in + context, do not read it again. +- **1.1.2** Apply every entry that matches the request. + - **1.1.2.1** An entry overrides a default of the skill and an + entry of the wiki's `MEMORY.md`, for work done with this skill + only. + - **1.1.2.2** An entry never overrides the boundaries in the wiki's + `AGENTS.md`; when the two conflict, follow the boundary and ask + the user. + - **1.1.2.3** An entry never changes a tool contract: a command, a + flag, or a required frontmatter field. +- **1.1.3** Every heading and list item is numbered as + [sectionize-markdown](../../skills/sectionize-markdown/sectionize-markdown.md) + describes. Cite an item as `SKILL_NAME MEMORIES.md item 2.1.3`; a + new item takes the next free number in its subsection. + +### 1.2 When to write + +- **1.2.1** Write an entry when one of these happens while the skill + is in use: + - **1.2.1.1** The user states a requirement or preference for this + skill that outlasts the current request ("always", "never", "from + now on", "when you do X"). + - **1.2.1.2** The user corrects the skill's output or a step it + took. + - **1.2.1.3** The same kind of request comes for the second time; + record the pattern and what the user wanted. + - **1.2.1.4** A run verifies a reusable fact about the skill's + inputs, hosts, or tools that the skill does not state. + - **1.2.1.5** The user explicitly asks you to remember something + for this skill. +- **1.2.2** Do not write: + - **1.2.2.1** One-off values that only apply to the current + request. + - **1.2.2.2** Passwords, tokens, cookies, personal data, or + anything the user asks to keep out. + - **1.2.2.3** Preferences that hold for any wiki work; those go to + the wiki's `MEMORY.md`. + - **1.2.2.4** Facts the skill or a wiki page already states; cite + them instead. + - **1.2.2.5** Guesses; only what the user said or a run showed. + +### 1.3 How to write + +- **1.3.1** Put the entry under the `###` subsection that fits. + - **1.3.1.1** When none fits, add a `###` subsection with the next + free number under the matching `##`, or a new `##` category with + the next free number. + - **1.3.1.2** Never put an entry directly under a `##`; its id + would collide with a later subsection. +- **1.3.2** Use the knowledge-entry shape of sectionize-markdown + section 4: a header line with the source tag and a short title, + one child per fact, and a last child with the ref and the date. +- **1.3.3** Source tags: `user` (the user said it), `run` (a command + or tool run showed it), `source` (a file under `raw/` states it), + joined with `+` when several agree. +- **1.3.4** The ref says where to check the fact: the user's request + and its date, the command and its output line, or a page or raw + path. +- **1.3.5** Write the entry in the same turn, then tell the user + which item was added or changed, for example "Saved to SKILL_NAME + MEMORIES.md item 2.1.1". +- **1.3.6** Bump `version` (patch for a changed entry, minor for a + new entry or subsection) and set `updated` in the frontmatter on + every change. +- **1.3.7** Append `- YYYY-MM-DD - SKILL_NAME memory: <item id> <title>` + to `wiki/log.md`, and bump that file too. +- **1.3.8** The change lands in the save of the current ingest cycle + or housekeeping pass. + - **1.3.8.1** With no cycle running, save it on its own with + `fiona-assistant git save --description "SKILL_NAME memory: <item id> <title>"`. + +An entry looks like this: + +```text +- **2.1.1** [user] "Reports open with the verdict" + - **2.1.1.1** The report starts with pass or fail, then the evidence. + - **2.1.1.2** Applies to every run of this skill. + - **2.1.1.3** Ref: user request YYYY-MM-DD; added YYYY-MM-DD. +``` + +### 1.4 How the memory evolves + +- **1.4.1** Before adding an entry, look for a near-duplicate in the + same category; extend it instead and keep its id and its earliest + date. +- **1.4.2** A finding that contradicts an entry becomes its next + child, titled `OVERRIDE: ...`, with the new facts and its own ref. +- **1.4.3** An entry that no longer applies is marked + **Superseded YYYY-MM-DD** with the reason; it is not deleted. +- **1.4.4** An entry that held for several uses graduates; mark it + **Promoted YYYY-MM-DD** with the target. + - **1.4.4.1** A rule for every run of the skill moves into + [the skill](./SKILL.md). + - **1.4.4.2** A preference that turns out to hold for all wiki + work moves to the wiki's `MEMORY.md`. +- **1.4.5** Never renumber ids; pages, log lines, and chats may cite + them. +- **1.4.6** Git is the memory's history: `git log -p MEMORIES.md` in + the skill's folder shows when each entry changed and why. + +## 2. Output and wording + +What the skill produces and how it reads. + +### 2.1 Style and format + +### 2.2 Placement and naming + +## 3. Inputs and sources + +What the skill reads and where it comes from. + +### 3.1 Where inputs come from + +### 3.2 Quirks and parsing + +## 4. Steps and tools + +How the skill's commands and the machine behave. + +### 4.1 Commands and flags + +### 4.2 Machine and shell + +## 5. Recurring requests + +Requests that come back, and what the user wanted each time. + +### 5.1 Patterns diff --git a/src/fiona/data/llm_wiki/utils/skills/memory/SKILL.md b/src/fiona/data/llm_wiki/utils/skills/memory/SKILL.md new file mode 100644 index 0000000..86bb7d5 --- /dev/null +++ b/src/fiona/data/llm_wiki/utils/skills/memory/SKILL.md @@ -0,0 +1,109 @@ +--- +name: memory +category: productivity +description: Give a skill its own memory, a MEMORIES.md beside its SKILL.md. +author: nvaldez +version: 1.0.0 +created: 2026-10-15 +updated: 2026-10-15 +--- + +# Skill Memory + +<!-- toc-start: headings 2..6 --> + +- [1. What a skill memory is](#1-what-a-skill-memory-is) +- [2. Give a skill memory](#2-give-a-skill-memory) +- [3. Wire it into the skill](#3-wire-it-into-the-skill) +- [4. Use a skill memory](#4-use-a-skill-memory) + +<!-- toc-end --> + +A skill with memory keeps what its users want from that skill across +sessions, in a `MEMORIES.md` file beside its `SKILL.md`. The blank +file to copy is [MEMORIES](./MEMORIES.md). The wiki-wide memory is +[MEMORY](../../../MEMORY.md); this skill does not replace it. + +Every heading and list item is numbered as +[sectionize-markdown](../sectionize-markdown/sectionize-markdown.md) +describes. Cite a rule as `memory skill item 2.3`. + +## 1. What a skill memory is + +- **1.1** One file per skill: `MEMORIES.md` in the skill's folder, + next to its `SKILL.md`. +- **1.2** It holds facts about operating that one skill: + - **1.2.1** Preferences for the skill's output. + - **1.2.2** Corrections to the skill's steps or results. + - **1.2.3** Request patterns that came back for that skill. + - **1.2.4** Facts a run verified about the skill's inputs, hosts, + and tools. +- **1.3** It does not hold: + - **1.3.1** Knowledge from sources; that is a page under `wiki/`. + - **1.3.2** Preferences for the whole wiki; those go to + [MEMORY](../../../MEMORY.md). + - **1.3.3** Passwords, tokens, cookies, or personal data. +- **1.4** Precedence, from strongest to weakest: + - **1.4.1** The boundaries in [AGENTS](../../../AGENTS.md) and tool + contracts (commands, flags, required frontmatter). + - **1.4.2** The skill's `MEMORIES.md`, for work done with that skill. + - **1.4.3** The wiki's `MEMORY.md`. + - **1.4.4** The defaults in the skill's `SKILL.md`. + +## 2. Give a skill memory + +- **2.1** Give a skill memory when the user asks for a skill with + memory, or when the same correction to one skill comes a second + time. +- **2.2** A new skill lives under `utils/skills_local/<skill>/`. A + skill shipped under `utils/skills/` can also get a memory; the + template never ships a `MEMORIES.md` for it, so an update leaves the + file alone. +- **2.3** Copy [MEMORIES](./MEMORIES.md) to + `<skill-folder>/MEMORIES.md`. Never edit the template copy in this + folder. +- **2.4** In the copy: + - **2.4.1** Set frontmatter `name` to `<skill>-memories`, `version` + to `1.0.0`, and `created` and `updated` to today. + - **2.4.2** Replace every `SKILL_NAME` with the skill's name. + - **2.4.3** Rename the empty categories in sections 2 to 5 to fit + the skill, or keep them. Leave section 1 as it is. +- **2.5** Add the memory section of section 3 to the skill's + `SKILL.md` and bump that file's `version`. +- **2.6** Append `- YYYY-MM-DD - memory: <skill> gets MEMORIES.md` to + `wiki/log.md`, then save with + `fiona-assistant git save --description "memory: <skill> gets MEMORIES.md"`. + +## 3. Wire it into the skill + +- **3.1** The skill's `SKILL.md` gets this section as its last + numbered section, with `N` set to that section's number: + +```markdown +## N. Memory + +- **N.1** After loading this skill, read [MEMORIES](./MEMORIES.md) + once per session. When it is already in context, do not read it + again. +- **N.2** Update it whenever an important memory about this skill + comes up: a lasting preference, a correction, a repeated request, + or a fact a run verified. Also update it whenever the user + explicitly asks you to remember something for this skill. + Section 1 of that file says how. +``` + +- **3.2** Keep the link relative (`./MEMORIES.md`) so the skill still + finds its memory after a move. + +## 4. Use a skill memory + +- **4.1** Section 1 of each skill's `MEMORIES.md` is the rule for + reading and writing it. This file does not repeat it. +- **4.2** Pick the narrowest memory that covers the fact: + - **4.2.1** It only matters while one skill runs: that skill's + `MEMORIES.md`. + - **4.2.2** It matters for any wiki work: `MEMORY.md`. + - **4.2.3** When unsure, ask the user which one. +- **4.3** When a request uses a skill that has no memory and a + memory comes up, write it to `MEMORY.md` and suggest giving the + skill one (section 2). diff --git a/src/fiona/data/llm_wiki/utils/skills/sectionize-markdown/sectionize-markdown.md b/src/fiona/data/llm_wiki/utils/skills/sectionize-markdown/sectionize-markdown.md new file mode 100644 index 0000000..2f06cf4 --- /dev/null +++ b/src/fiona/data/llm_wiki/utils/skills/sectionize-markdown/sectionize-markdown.md @@ -0,0 +1,157 @@ +--- +name: sectionize-markdown +version: 1.1.2 +title: How to sectionize a markdown file +created: '2026-10-14' +updated: '2026-10-14' +confidence: high +llm: + - model: Claude Opus 5.5 +--- + +# How to sectionize a markdown file + +<!-- toc-start: headings 2..6 --> + +- [1. Headings](#1-headings) +- [2. List item ids](#2-list-item-ids) +- [3. Splitting long lines](#3-splitting-long-lines) +- [4. Knowledge entries](#4-knowledge-entries) +- [5. Organize by category](#5-organize-by-category) +- [6. Doing it safely](#6-doing-it-safely) +- [7. Example](#7-example) + +<!-- toc-end --> + +Numbers every heading and every list item of a markdown file so each +rule, step, or fact has a stable id that can be cited ("item +4.2.3"), tracked, and checked for duplicates. It suits rule files, +skill instructions, glossaries, and knowledge caches. + +## 1. Headings + +- **1.1** Number every heading below the title: `## N.` for sections, + `### N.M` for subsections, `#### N.M.K` below that. +- **1.2** The `#` title and the table of contents are not numbered. +- **1.3** Put a TOC of markdown links right under the title and intro. + Regenerate it whenever headings change. +- **1.4** Anchors are GitHub-style slugs: `## 5.6 Download the logs` + becomes `#56-download-the-logs`. +- **1.5** Avoid em dashes and other punctuation in headings; use a colon + (`10.7 Outcome: a match exists`). Punctuation makes the slug + unpredictable and breaks TOC links (MD051). +- **1.6** A label that only introduces a list ("Files in this + folder:") is a numbered list item, not a heading. + +## 2. List item ids + +- **2.1** Every list item gets a bold id right after the dash: + `- **4.2.1** Text`. Bold is two asterisks on each side; a single + asterisk is italic. +- **2.2** The id is the section number plus a running counter: + `**<section>.<n>**`. Nested items extend the parent id: + `**<section>.<n>.<m>**`, indented two spaces. +- **2.3** The counter runs through the whole section, across separate + lists split by paragraphs or code blocks. Two lists under `### 12.3` + give `**12.3.1**`…`**12.3.3**`, then `**12.3.4**`…, never a second + `**12.3.1**`. +- **2.4** Checklists keep the box first: `- [ ] **13.1** Verify …`. +- **2.5** Do not number the TOC, code blocks, or table rows. +- **2.6** State the convention in the file's first section so the next + editor follows it ("Cite an item as `item 4.2.3`"). +- **2.7** Never renumber existing ids; other notes may cite them. A new + item takes the next free number in its section, even if that puts it + out of reading order. Fix the order only while the ids are still + unreleased (uncommitted). + +## 3. Splitting long lines + +- **3.1** A list item holding several facts becomes a short header item + plus one child per fact. +- **3.2** One child is one sentence, or two closely linked sentences. +- **3.3** Split at sentence ends (`.`, `?` or `!`, then a space and an + uppercase letter, backtick, `*`, `"` or `[`). Do not split inside + backticks or parentheses, or after `e.g.`, `i.e.`, `vs.`, `cf.`. +- **3.4** Merge fragments shorter than about 45 characters back into the + previous child; they read better together. +- **3.5** A long single sentence stays on one line. Do not cut a + sentence in half to meet a length target. +- **3.6** Leave command lines, code blocks, and table rows as they are. + +## 4. Knowledge entries + +A file that collects findings or lessons learned uses a fixed entry +shape so each fact is citable and the source is visible: + +```text +- **<section>.<n>** [<source>] "<short title>" + - **<section>.<n>.1** <reusable fact + where it applies> + - **<section>.<n>.2** <next fact> + - **<section>.<n>.<k>** Ref: <path:line or query>; added <YYYY-MM-DD>. +``` + +- **4.1** The entry line holds only the source tag and the title. +- **4.2** `<source>` names where the fact was verified (a repository, a + spec, …), joined with `+` when several agree. +- **4.3** The last child is always the ref with the date it was added. +- **4.4** A finding that contradicts an entry becomes that entry's next + child, titled `OVERRIDE: …`, with its own children. Or mark the old + entry **Superseded YYYY-MM-DD** with the reason. + +## 5. Organize by category + +- **5.1** Group entries by topic (setup, build, deployment, …), not by + the date they were added. The date lives in the ref line. +- **5.2** Grouping puts near-duplicates next to each other. Merge them + into one entry and keep the earliest `added` date. +- **5.3** A glossary gets the same treatment: one + `###` per term under a `##` category, its definition split into + numbered items. + +## 6. Doing it safely + +- **6.1** Restructure with a script, not by hand, when the file has more + than a few dozen items. Hand edits drop or reorder text. +- **6.2** The script asserts that the words are preserved: same word + sequence before and after, only ids and line breaks added. +- **6.3** Verify after every pass: + - **6.3.1** No duplicate ids (`^\s*- (\[ \] )?\*\*([\d.]+)\*\*`). + - **6.3.2** Every `](#anchor)` resolves to a heading slug. + - **6.3.3** The entry count matches the original; diff titles against + `HEAD` when it does not (an uncommitted entry may explain it). + - **6.3.4** One `Ref:` line per entry. +- **6.4** Watch for editors that reformat on save (escaped `\*`, `*` + bullets padded with three spaces, lost HTML comments, fences without a language). Normalize + back and re-verify. +- **6.5** On Windows PowerShell, write helper scripts to `%TEMP%` and run + them; `python -c` with backticks breaks. Avoid printing non-ASCII to + the cp1252 console. Delete the temp scripts afterwards. +- **6.6** Keep Unix line endings and `-` bullets, and run the markdown + linter before committing. + +## 7. Example + +Before: + +```text +## Topic + +- Rule about the topic. A fact about it. How to fix it. Ref: file:line. +- Label for a group: first part of the rule, second part of the rule. +``` + +After: + +```text +## 3. Topic + +### 3.2 Subtopic + +- **3.2.4** [source] "Rule about the topic" + - **3.2.4.1** A fact about it. + - **3.2.4.2** How to fix it. + - **3.2.4.3** Ref: file:line; added YYYY-MM-DD. +- **3.2.5** Label for a group: + - **3.2.5.1** First part of the rule. + - **3.2.5.2** Second part of the rule. +``` diff --git a/src/fiona/librarian/toc_generator.py b/src/fiona/librarian/toc_generator.py index e42e0fc..5a642ce 100644 --- a/src/fiona/librarian/toc_generator.py +++ b/src/fiona/librarian/toc_generator.py @@ -94,9 +94,12 @@ def depth_setting(settings: dict, key: str) -> int | None: def slugify(text: str) -> str: - """Convert a header text into a GitHub-style anchor.""" + """Convert a header text into a GitHub-style anchor (SPEC 7.3.1.1.3). + + Each space becomes one hyphen; runs are kept so ``a | b`` is ``a--b``. + """ slug = re.sub(r"[^\w\s-]", "", text.lower()) - return re.sub(r"\s+", "-", slug.strip()) + return re.sub(r"\s", "-", slug.strip()) def excluded(text: str, patterns: list[typing.Pattern]) -> bool: diff --git a/src/fiona/run_tests.py b/src/fiona/run_tests.py index 9eb2fbf..c595654 100644 --- a/src/fiona/run_tests.py +++ b/src/fiona/run_tests.py @@ -452,7 +452,7 @@ class TestRunner: Args: test_path: Path to the test file or directory verbose: Whether to run in verbose mode - capture_output: Whether to suppress the pytest output (for CI) + capture_output: When True, hide pytest stdout/stderr (default False) with_coverage: Whether to run with coverage (default: True) ignore: Pattern to ignore (e.g., "test_sanity.py") @@ -494,7 +494,10 @@ class TestRunner: completed = subprocess.run(cmd, capture_output=True, text=True, check=False, env=env) if not capture_output: - print(completed.stdout) + if completed.stdout: + print(completed.stdout, end="" if completed.stdout.endswith("\n") else "\n") + if completed.stderr: + print(completed.stderr, end="" if completed.stderr.endswith("\n") else "\n", file=sys.stderr) logger.debug("pytest command: %s", " ".join(str(part) for part in cmd)) logger.debug("pytest return code: %s", completed.returncode) @@ -606,29 +609,56 @@ def main(): """Main function for command-line execution.""" parser = argparse.ArgumentParser(description="Run Fiona tools tests.") parser.add_argument("--verbose", action="store_true", help="Run tests in verbose mode") - parser.add_argument("--ci", action="store_true", help="Run in CI mode (capture output)") + parser.add_argument( + "--ci", + action="store_true", + help="Non-interactive CI mode: verbose pytest with full output to the log", + ) + parser.add_argument( + "--non-interactive", + action="store_true", + help="Alias for --ci", + ) parser.add_argument("--test", help="Run specific test file or directory") parser.add_argument("--skip-cov", action="store_true", help="Skip coverage") args = parser.parse_args() + # CURSOR: use `is True` — bool(MagicMock()) is True in unit tests. + ci_mode = args.ci is True or args.non_interactive is True + verbose = args.verbose is True or ci_mode + capture_output = False # Initialize test runner runner = TestRunner() # Determine if we should run with coverage - with_coverage = not args.skip_cov + with_coverage = args.skip_cov is not True + + def _print_failures() -> None: + """Echo parsed failure details after pytest output (CI-friendly).""" + for detail in runner.test_results.values(): + for entry in detail.get("failed_tests") or []: + test_id = entry.get("id", "?") + message = entry.get("message", "") + print(f"FAILED {test_id} - {message}", file=sys.stderr) + if entry.get("location"): + print(f" at {entry['location']}", file=sys.stderr) + if entry.get("assert"): + print(f" {entry['assert']}", file=sys.stderr) if args.test: # Run specific test - result = runner.run_pytest(args.test, args.verbose, args.ci, with_coverage) + result = runner.run_pytest(args.test, verbose, capture_output, with_coverage) if not result["passed"]: - print(f"Test failed: {args.test}") + _print_failures() + print(f"Test failed: {args.test}", file=sys.stderr) sys.exit(1) else: # Run all tests - all_passed = runner.run_all_tests(args.verbose, args.ci, with_coverage) + all_passed = runner.run_all_tests(verbose, capture_output, with_coverage) if not all_passed: - print("Some tests failed") #TODO: this is not very informative, need improving + _print_failures() + print("Some tests failed", file=sys.stderr) sys.exit(1) # Print summary diff --git a/src/fiona/wiki/ingest/converters/__init__.py b/src/fiona/wiki/ingest/converters/__init__.py index 698695d..dd36393 100644 --- a/src/fiona/wiki/ingest/converters/__init__.py +++ b/src/fiona/wiki/ingest/converters/__init__.py @@ -1,6 +1,7 @@ """Shipped converters, in registration order. Generated by AI/LLM: 2026-09-30 - markdown, text, html, then pdf. +Modified by AI/LLM: 2026-10-07 - the html converter takes the block sweep settings. """ from fiona.wiki.ingest.converters.base import ConverterRegistry @@ -8,19 +9,30 @@ from fiona.wiki.ingest.converters.html import HtmlConverter from fiona.wiki.ingest.converters.markdown import MarkdownConverter from fiona.wiki.ingest.converters.pdf import PdfConverter from fiona.wiki.ingest.converters.text import TextConverter +from fiona.wiki.ingest.settings import IngestSettings, default_settings -def default_registry() -> ConverterRegistry: +def default_registry(settings: IngestSettings | None = None) -> ConverterRegistry: """Return the converters shipped with wiki ingest. Generated by AI/LLM: 2026-09-30 - registration order from SPEC 7.1.5. + Modified by AI/LLM: 2026-10-07 - ``html.*`` settings reach ``HtmlConverter``. + + Args: + settings: Ingest settings. ``None`` uses the defaults. Returns: ConverterRegistry: Markdown, text, HTML, and PDF. """ + settings = settings or default_settings() registry = ConverterRegistry() registry.register(MarkdownConverter) registry.register(TextConverter) - registry.register(HtmlConverter) + registry.register( + HtmlConverter( + min_words=settings.html_min_words, + max_link_density=settings.html_max_link_density, + ) + ) registry.register(PdfConverter) return registry diff --git a/src/fiona/wiki/ingest/converters/html.py b/src/fiona/wiki/ingest/converters/html.py index 7e23c00..cc67c9e 100644 --- a/src/fiona/wiki/ingest/converters/html.py +++ b/src/fiona/wiki/ingest/converters/html.py @@ -1,6 +1,7 @@ """HTML source converter. Generated by AI/LLM: 2026-09-30 - article text via beautifulsoup4 and markdownify. +Modified by AI/LLM: 2026-10-07 - the block sweep replaces the first-``<article>`` root. """ from __future__ import annotations @@ -9,9 +10,9 @@ import typing import urllib.parse import bs4 -import markdownify from fiona.wiki.ingest.converters.base import SourceConverter +from fiona.wiki.ingest.converters.html_sweep import sweep, unwrap_shadow_roots from fiona.wiki.ingest.media import decode_bytes from fiona.wiki.ingest.models import ConversionResult, FetchedSource @@ -23,6 +24,7 @@ class HtmlConverter(SourceConverter): """Convert an HTML document to markdown. Generated by AI/LLM: 2026-09-30 - SPEC 7.1.5.4.5. + Modified by AI/LLM: 2026-10-07 - sweep limits are constructor arguments. """ name: typing.ClassVar[str] = "html" @@ -30,10 +32,23 @@ class HtmlConverter(SourceConverter): extensions: typing.ClassVar[tuple[str, ...]] = (".html", ".htm", ".xhtml") raw_category: typing.ClassVar[str] = "articles" + def __init__(self, min_words: int = 4, max_link_density: float = 0.5) -> None: + """Store the block sweep limits. + + Generated by AI/LLM: 2026-10-07 - defaults from SPEC 5.1 ``fiona.wiki.ingest.html``. + + Args: + min_words: Fewer words make a ``p`` or loose block ``short``. + max_link_density: Link text over text length above this is ``links``. + """ + self.min_words = min_words + self.max_link_density = max_link_density + def convert(self, source: FetchedSource) -> ConversionResult: - """Parse HTML and return the article body as markdown. + """Parse HTML and return every swept block of the body as markdown. Generated by AI/LLM: 2026-09-30 - drop scripts and absolutize links. + Modified by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6 block sweep over ``<body>``. Args: source: Archived HTML file. ``origin`` is the page URL when fetched. @@ -43,30 +58,27 @@ class HtmlConverter(SourceConverter): """ text, warnings = decode_bytes(source.path.read_bytes(), source.charset) soup = bs4.BeautifulSoup(text, "html.parser") + unwrap_shadow_roots(soup) for tag in soup.find_all(_DROP_TAGS): tag.decompose() for comment in soup.find_all(string=lambda node: isinstance(node, bs4.Comment)): comment.extract() - root = soup.find("article") or soup.find("main") or soup.body or soup + body = soup.body or soup origin = source.origin if source.origin_kind == "url" else None - _absolutize(root, soup, origin) - visible = root.get_text(strip=True) - heading = getattr(markdownify, "ATX", "ATX") - body = markdownify.markdownify(str(root), heading_style=heading, bullets="-").strip() - if body: - body += "\n" - meta = _source_meta(soup) - title = _html_title(soup, root) + _absolutize(body, soup, origin) conversion = "complete" - if not visible: + if not body.get_text(strip=True): conversion = "partial" warnings = [*warnings, _NO_TEXT] + swept = sweep(body, min_words=self.min_words, max_link_density=self.max_link_density) + if swept.partial: + conversion = "partial" return ConversionResult( - body=body, - title=title, - source_meta=meta, + body=swept.body, + title=_html_title(soup, body), + source_meta=_source_meta(soup), conversion=conversion, - warnings=warnings, + warnings=[*warnings, *swept.warnings], converter=self.name, ) @@ -89,19 +101,20 @@ def _absolutize(root: bs4.element.Tag, soup: bs4.BeautifulSoup, origin: str | No def _html_title(soup: bs4.BeautifulSoup, root: bs4.element.Tag) -> str | None: - """Return ``<title>``, then ``og:title``, then the first ``<h1>``. + """Return ``<title>``, then ``og:title``, then the first ``<h1>`` that has text. Generated by AI/LLM: 2026-09-30 - SPEC 7.1.5.4.5.1. + Modified by AI/LLM: 2026-10-07 - the root is ``<body>``, so a blank ``<h1>`` is skipped. """ if soup.title and soup.title.string and soup.title.string.strip(): return soup.title.string.strip() og = soup.find("meta", attrs={"property": "og:title"}) if og is not None and og.get("content"): return str(og["content"]).strip() or None - heading = root.find("h1") - if heading is not None: + for heading in root.find_all("h1"): text = heading.get_text(strip=True) - return text or None + if text: + return text return None diff --git a/src/fiona/wiki/ingest/converters/html_sweep.py b/src/fiona/wiki/ingest/converters/html_sweep.py new file mode 100644 index 0000000..0680fd0 --- /dev/null +++ b/src/fiona/wiki/ingest/converters/html_sweep.py @@ -0,0 +1,285 @@ +"""Score every block of an HTML body as ``keep``, ``review``, or ``drop``. + +Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6. Rules read HTML structure only. +""" + +from __future__ import annotations + +import dataclasses +import re +import typing + +import bs4 +import markdownify + +_BLOCK_KINDS = { + "h1": "heading", + "h2": "heading", + "h3": "heading", + "h4": "heading", + "h5": "heading", + "h6": "heading", + "p": "p", + "ul": "list", + "ol": "list", + "dl": "list", + "blockquote": "quote", + "pre": "pre", + "table": "table", + "figure": "figure", +} +_INLINE_TAGS = frozenset(( + "a", "abbr", "b", "bdi", "br", "cite", "code", "data", "del", "dfn", "em", + "font", "i", "img", "ins", "kbd", "label", "mark", "q", "s", "samp", "small", + "span", "strong", "sub", "sup", "time", "u", "var", "wbr", +)) +_LAYOUT_CELL_CHILDREN = (*_BLOCK_KINDS, "div") +_SKIPPED_TAGS = frozenset(("head",)) +_DROP_ANCESTORS = frozenset(("nav", "button", "select")) +_CHROME_TAGS = frozenset(("aside", "form", "dialog")) +_CHROME_ROLES = frozenset(("navigation", "banner", "contentinfo", "complementary")) +_HIDDEN_STYLE = re.compile(r"display\s*:\s*none|visibility\s*:\s*hidden", re.IGNORECASE) +_SHORT_KINDS = frozenset(("p", "loose")) +_ALL_REVIEW = "no block passed the content checks; every block is marked for review" + + +@dataclasses.dataclass +class Block: + """One swept block: an element, or a run of loose text and inline nodes. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.1 and 7.1.5.4.5.6.2. + """ + + nodes: list[bs4.PageElement] + kind: str + text: str = "" + link_chars: int = 0 + words: int = 0 + verdict: str = "" + reason: str = "" + + +@dataclasses.dataclass +class SweepResult: + """Markdown body, warnings, and whether every block needs review. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.6 and 7.1.5.4.5.6.7. + """ + + body: str + warnings: list[str] + partial: bool + + +def unwrap_shadow_roots(soup: bs4.BeautifulSoup) -> None: + """Read each declarative shadow root in place, so its text is swept. + + Generated by AI/LLM: 2026-10-07 - ``template[shadowrootmode]``; other templates are dropped later. + + Args: + soup: Parsed document, before the drop loop. + """ + for template in soup.find_all("template", attrs={"shadowrootmode": True}): + template.unwrap() + + +def sweep(root: bs4.element.Tag, *, min_words: int, max_link_density: float) -> SweepResult: + """Walk ``root``, classify each block, and render the kept and review blocks. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6. No clock, network, or random source. + + Args: + root: ``<body>``, or the document when there is no body. + min_words: ``fiona.wiki.ingest.html.min_words``. + max_link_density: ``fiona.wiki.ingest.html.max_link_density``. + + Returns: + SweepResult: Markdown body, the ``html sweep:`` warning, and the partial flag. + """ + blocks: list[Block] = [] + _walk(root, blocks) + seen: set[str] = set() + for block in blocks: + _measure(block) + block.verdict, block.reason = _classify(block, seen, min_words, max_link_density) + if block.verdict != "drop": + seen.add(block.text.casefold()) + counts = {verdict: sum(block.verdict == verdict for block in blocks) for verdict in ("keep", "review", "drop")} + warnings = [f"html sweep: {counts['keep']} keep, {counts['review']} review, {counts['drop']} drop"] + partial = counts["keep"] == 0 and counts["review"] > 0 + if partial: + warnings.append(_ALL_REVIEW) + parts = [part for part in (_render(block) for block in blocks if block.verdict != "drop") if part] + body = "\n\n".join(parts) + return SweepResult(body=body + "\n" if body else "", warnings=warnings, partial=partial) + + +def _walk(container: bs4.element.Tag, blocks: list[Block]) -> None: + """Append the blocks of ``container`` in document order. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.1 to 7.1.5.4.5.6.1.2. + """ + run: list[bs4.PageElement] = [] + for child in list(container.children): + if not isinstance(child, bs4.element.Tag): + if isinstance(child, bs4.element.NavigableString): + run.append(child) + continue + if child.name in _SKIPPED_TAGS: + continue + if _is_block(child): + _flush(run, blocks) + blocks.append(Block(nodes=[child], kind=_BLOCK_KINDS[child.name])) + elif child.name in _INLINE_TAGS and not _holds_structure(child): + run.append(child) + else: + _flush(run, blocks) + _walk(child, blocks) + _flush(run, blocks) + + +def _flush(run: list[bs4.PageElement], blocks: list[Block]) -> None: + """Close a loose-text run as one block, unless it holds no text. + + Generated by AI/LLM: 2026-10-07 - whitespace between tags is not a block. + """ + if run and _fold(_raw_text(run)): + blocks.append(Block(nodes=list(run), kind="loose")) + run.clear() + + +def _is_block(tag: bs4.element.Tag) -> bool: + """Return whether ``tag`` is a block. A layout table is a container instead. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.1 layout tables. + """ + if tag.name not in _BLOCK_KINDS: + return False + if tag.name != "table": + return True + return not any(cell.find(_LAYOUT_CELL_CHILDREN) for cell in tag.find_all(("td", "th"))) + + +def _holds_structure(tag: bs4.element.Tag) -> bool: + """Return whether an inline tag wraps a block or container, like a card link. + + Generated by AI/LLM: 2026-10-07 - ``<a><div>...</div></a>`` is walked as a container. + """ + return tag.find(lambda node: node.name not in _INLINE_TAGS) is not None + + +def _raw_text(nodes: typing.Iterable[bs4.PageElement]) -> str: + """Return the text of ``nodes`` with no separator and ``<br>`` as a space. + + Generated by AI/LLM: 2026-10-07 - keeps ``<b>Fa</b>st`` as one word. + """ + parts: list[str] = [] + for node in nodes: + items = [node] if isinstance(node, bs4.element.NavigableString) else [node, *node.descendants] + for item in items: + if isinstance(item, bs4.element.NavigableString): + parts.append(str(item)) + elif item.name == "br": + parts.append(" ") + return "".join(parts) + + +def _fold(text: str) -> str: + """Fold whitespace runs to one space and trim. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.2. + """ + return " ".join(text.split()) + + +def _measure(block: Block) -> None: + """Set the folded text, link text length, and word count of a block. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.2. + """ + block.text = _fold(_raw_text(block.nodes)) + links: list[bs4.element.Tag] = [] + for node in block.nodes: + if isinstance(node, bs4.element.Tag): + links.extend([node] if node.name == "a" else node.find_all("a")) + block.link_chars = sum(len(_fold(_raw_text([link]))) for link in links) + block.words = len(block.text.split()) + + +def _classify(block: Block, seen: set[str], min_words: int, max_link_density: float) -> tuple[str, str]: + """Return ``(verdict, reason)`` with the first rule that holds. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.3 to 7.1.5.4.5.6.5. + """ + folded = block.text.casefold() + if not any(char.isalnum() for char in block.text): + return "drop", "" + if block.words < min_words and folded in seen: + return "drop", "" + if _inside(block, _drops): + return "drop", "" + if _inside(block, _is_chrome): + return "review", "chrome" + if folded in seen: + return "review", "repeat" + if block.link_chars / len(block.text) > max_link_density: + return "review", "links" + if block.kind in _SHORT_KINDS and block.words < min_words: + return "review", "short" + return "keep", "" + + +def _inside(block: Block, predicate: typing.Callable[[bs4.element.Tag], bool]) -> bool: + """Return whether the block, or an ancestor below the document, matches. + + Generated by AI/LLM: 2026-10-07 - an element block checks itself; a loose run starts at its parent. + """ + first = block.nodes[0] + node = first.parent if block.kind == "loose" else first + while isinstance(node, bs4.element.Tag) and not isinstance(node, bs4.BeautifulSoup): + if predicate(node): + return True + node = node.parent + return False + + +def _drops(tag: bs4.element.Tag) -> bool: + """Return whether text under ``tag`` is never content: nav, controls, hidden. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.3. + """ + if tag.name in _DROP_ANCESTORS or tag.has_attr("hidden"): + return True + if str(tag.get("aria-hidden", "")).strip().lower() == "true": + return True + return bool(_HIDDEN_STYLE.search(str(tag.get("style", "")))) + + +def _is_chrome(tag: bs4.element.Tag) -> bool: + """Return whether ``tag`` is page chrome: aside, form, landmark role, outer header or footer. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.4 ``chrome``. + """ + if tag.name in _CHROME_TAGS: + return True + if str(tag.get("role", "")).strip().lower() in _CHROME_ROLES: + return True + if tag.name in ("header", "footer"): + return tag.find_parent(("article", "main")) is None + return False + + +def _render(block: Block) -> str: + """Convert one block to markdown and wrap a ``review`` block in its markers. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.4.5.6.6, options of 7.1.5.4.5.4. + """ + markup = "".join( + node.output_ready(formatter="minimal") if isinstance(node, bs4.element.NavigableString) else str(node) + for node in block.nodes + ) + heading = getattr(markdownify, "ATX", "ATX") + text = markdownify.markdownify(markup, heading_style=heading, bullets="-").strip() + if not text or block.verdict != "review": + return text + return f"<!-- fiona:review {block.reason} -->\n\n{text}\n\n<!-- /fiona:review -->" diff --git a/src/fiona/wiki/ingest/fetchers.py b/src/fiona/wiki/ingest/fetchers.py index 46a47c9..2b477c0 100644 --- a/src/fiona/wiki/ingest/fetchers.py +++ b/src/fiona/wiki/ingest/fetchers.py @@ -1,10 +1,12 @@ """Fetch a local file or an HTTP URL. Generated by AI/LLM: 2026-09-30 - SPEC 7.1.5.2. No credentials are sent. +Modified by AI/LLM: 2026-10-07 - login detection, SPEC 7.1.5.2.8. """ from __future__ import annotations +import http import logging import pathlib import re @@ -20,13 +22,15 @@ from fiona.wiki.ingest.media import ( slug_from_url, utc_now, ) -from fiona.wiki.ingest.models import FetchedSource, IngestError +from fiona.wiki.ingest.models import AuthRequiredError, FetchedSource, IngestError from fiona.wiki.ingest.settings import IngestSettings, default_settings logger = logging.getLogger(__name__) _SCHEME = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*)://") _WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:[\\/]") +_AUTH_STATUSES = (401, 403) +_PASSWORD_INPUT = re.compile(rb"<input\b[^>]*?\btype\s*=\s*[\"']?password\b", re.IGNORECASE) def url_scheme(argument: str) -> str | None: @@ -199,13 +203,17 @@ class UrlFetcher(SourceFetcher): IngestError: The response is not 2xx, the network fails, or the body is too large. """ settings = self.settings or default_settings() + requested = without_userinfo(argument) request = urllib.request.Request( - _without_userinfo(argument), + requested, headers={"User-Agent": settings.user_agent}, ) try: response = self.opener.open(request, timeout=settings.timeout_seconds) except urllib.error.HTTPError as exc: + if exc.code in _AUTH_STATUSES: + phrase = http.HTTPStatus(exc.code).phrase + raise AuthRequiredError(requested, f"{exc.code} {phrase}") from exc raise IngestError(f"Download failed: {argument}: {exc.code} {exc.reason}") from exc except urllib.error.URLError as exc: raise IngestError(f"Download failed: {argument}: {exc.reason}") from exc @@ -216,6 +224,7 @@ class UrlFetcher(SourceFetcher): parsed = urllib.parse.urlsplit(argument) suffix = pathlib.PurePosixPath(parsed.path).suffix media, charset = detect_media_type(body, suffix, header, self.registry) + _check_login(requested, final_url, media, body, settings.login_url_patterns) extension = extension_for(media, suffix, self.registry) slug = slug_from_url(argument) dest_dir.mkdir(parents=True, exist_ok=True) @@ -244,10 +253,41 @@ class UrlFetcher(SourceFetcher): ) -def _without_userinfo(url: str) -> str: +def _check_login( + requested: str, + final_url: str, + media: str, + body: bytes, + patterns: list[str], +) -> None: + """Raise when a 2xx answer is a login redirect or a login form. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.2.8.2 redirect, then login form. + + Args: + requested: URL that was requested, without userinfo. + final_url: URL after redirects. + media: Detected media type of the body. + body: Response body, already within the size limit. + patterns: ``login_url_patterns`` from the settings. + + Raises: + AuthRequiredError: The answer is a login page, not the source. + """ + if final_url and final_url != requested: + parts = urllib.parse.urlsplit(final_url) + target = f"{parts.hostname or ''}{parts.path}" + if any(re.search(pattern, target, re.IGNORECASE) for pattern in patterns): + raise AuthRequiredError(requested, f"redirected to {final_url}") + if media == "text/html" and _PASSWORD_INPUT.search(body): + raise AuthRequiredError(requested, f"login page at {final_url or requested}") + + +def without_userinfo(url: str) -> str: """Drop userinfo so the request sends no credentials. Generated by AI/LLM: 2026-09-30 - SPEC 7.1.5.2.2. + Modified by AI/LLM: 2026-10-07 - public; also used for ``--origin``. """ parts = urllib.parse.urlsplit(url) if parts.username is None and parts.password is None: diff --git a/src/fiona/wiki/ingest/log_writer.py b/src/fiona/wiki/ingest/log_writer.py index b569549..b623f78 100644 --- a/src/fiona/wiki/ingest/log_writer.py +++ b/src/fiona/wiki/ingest/log_writer.py @@ -44,7 +44,7 @@ class IngestLog: Args: day: UTC date ``YYYY-MM-DD``. - origin: URL or resolved file path. + origin: URL, or file path (wiki-relative inside the wiki). raw_rel: Raw path relative to the wiki root. page_rel: Page path relative to the wiki root, or None. dry_run: When true, leave the log untouched. diff --git a/src/fiona/wiki/ingest/models.py b/src/fiona/wiki/ingest/models.py index 2d91f29..2e835b3 100644 --- a/src/fiona/wiki/ingest/models.py +++ b/src/fiona/wiki/ingest/models.py @@ -26,6 +26,24 @@ class IngestError(Exception): self.exit_code = exit_code +class AuthRequiredError(IngestError): + """Raised when a URL answers with a login instead of the source. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.2.8. Exit 4, nothing written. + """ + + def __init__(self, url: str, reason: str) -> None: + """Store the URL and the reason for the ``Error:`` line. + + Args: + url: The URL the user passed, without userinfo. + reason: ``<status> <phrase>``, ``redirected to <url>``, or ``login page at <url>``. + """ + super().__init__(f"Authentication required: {url}: {reason}", exit_code=4) + self.url = url + self.reason = reason + + @dataclasses.dataclass class FetchedSource: """A local file plus the facts the later stages need. diff --git a/src/fiona/wiki/ingest/pipeline.py b/src/fiona/wiki/ingest/pipeline.py index 708538a..3123d08 100644 --- a/src/fiona/wiki/ingest/pipeline.py +++ b/src/fiona/wiki/ingest/pipeline.py @@ -18,16 +18,23 @@ from fiona.librarian.md_frontmatter import FrontmatterError, parse_frontmatter from fiona.wiki.ingest.archiver import RawArchiver from fiona.wiki.ingest.converters import default_registry from fiona.wiki.ingest.converters.base import ConversionError, ConverterRegistry -from fiona.wiki.ingest.fetchers import LocalFileFetcher, SourceFetcher, UrlFetcher, url_scheme +from fiona.wiki.ingest.fetchers import ( + LocalFileFetcher, + SourceFetcher, + UrlFetcher, + url_scheme, + without_userinfo, +) from fiona.wiki.ingest.log_writer import IngestLog from fiona.wiki.ingest.media import slug_from_file, slug_from_url, utc_day -from fiona.wiki.ingest.models import FetchedSource, IngestError, IngestResult +from fiona.wiki.ingest.models import AuthRequiredError, FetchedSource, IngestError, IngestResult from fiona.wiki.ingest.page_writer import RawPageWriter from fiona.wiki.ingest.settings import IngestSettings, load_ingest_settings logger = logging.getLogger(__name__) _NO_CONVERTER = "No converter for {media}. Convert {raw} by hand." +_AUTH_NEXT = "Next: save it into ingest/, then run: wiki ingest <file> --origin {url}" class IngestPipeline: @@ -74,10 +81,13 @@ class IngestPipeline: raw_only: bool = False, force: bool = False, dry_run: bool = False, + origin: str | None = None, ) -> IngestResult: """Ingest one URL or local file. Generated by AI/LLM: 2026-09-30 - exit 1 raises; exit 0 and 3 return a result. + Modified by AI/LLM: 2026-10-07 - ``origin`` for a file saved by hand (SPEC 7.1.5.2.9). + Modified by AI/LLM: 2026-10-08 - a file inside the wiki has a relative origin (SPEC 7.1.5.2.10). Args: argument: Source as given. @@ -86,16 +96,19 @@ class IngestPipeline: raw_only: Archive and record, skip convert and write. force: Replace an existing page, including a duplicate. dry_run: Report the run and write nothing under the wiki. + origin: ``--origin`` URL a local file was saved from. Returns: IngestResult: Facts for stdout. ``exit_code`` is 0 or 3. Raises: IngestError: Exit 1. Nothing new remains under the wiki. + AuthRequiredError: Exit 4. The URL needs a login. """ scheme = url_scheme(argument) if scheme is not None and scheme.lower() not in {"http", "https"}: raise IngestError(f"Unsupported URL scheme: {scheme}") + _check_origin(argument, scheme, origin) slug = slug_from_url(argument) if scheme else slug_from_file(pathlib.Path(argument)) page = self.page_writer.page_path(slug, self.settings, target) if page.is_file() and not force: @@ -113,6 +126,18 @@ class IngestPipeline: else: dest = self.wiki_root / "ingest" fetched = fetcher.fetch(argument, dest) + if fetched.origin_kind == "file" and fetched.path.is_relative_to(self.wiki_root.resolve()): + fetched = dataclasses.replace( + fetched, + origin=fetched.path.relative_to(self.wiki_root.resolve()).as_posix(), + ) + if origin is not None: + fetched = dataclasses.replace( + fetched, + origin=without_userinfo(origin), + origin_kind="url", + final_url=None, + ) archive = self.archiver.archive( fetched, registry=self.registry, @@ -297,6 +322,7 @@ def default_pipeline(wiki_root: pathlib.Path) -> IngestPipeline: """Build the pipeline with the shipped fetchers and converters. Generated by AI/LLM: 2026-09-30 - URL fetcher, then local file fetcher. + Modified by AI/LLM: 2026-10-07 - one settings object for the registry and the pipeline. Args: wiki_root: Wiki root directory. @@ -304,7 +330,8 @@ def default_pipeline(wiki_root: pathlib.Path) -> IngestPipeline: Returns: IngestPipeline: Ready to ``run`` or ``list_pending``. """ - registry = default_registry() + settings = load_ingest_settings(wiki_root) + registry = default_registry(settings) return IngestPipeline( wiki_root=wiki_root, fetchers=[UrlFetcher(registry), LocalFileFetcher(registry)], @@ -312,7 +339,7 @@ def default_pipeline(wiki_root: pathlib.Path) -> IngestPipeline: archiver=RawArchiver(wiki_root), page_writer=RawPageWriter(wiki_root), log_writer=IngestLog(wiki_root), - settings=load_ingest_settings(wiki_root), + settings=settings, ) @@ -343,6 +370,11 @@ def add_ingest_arguments(parser: argparse.ArgumentParser) -> None: help="llm-wiki directory (default: .)", ) parser.add_argument("-r", "--raw", dest="raw", help="raw path, relative to raw/") + parser.add_argument( + "--origin", + dest="origin", + help="http(s) URL a local file was saved from", + ) parser.add_argument( "-t", "--target", @@ -382,7 +414,7 @@ def ingest_main(argv: list[str] | None = None) -> int: argv: Arguments after the subcommand. ``None`` reads the process arguments. Returns: - int: 0, 1, 2, or 3. + int: 0, 1, 2, 3, or 4. """ parser = argparse.ArgumentParser(prog="fiona-assistant wiki ingest") add_ingest_arguments(parser) @@ -399,7 +431,7 @@ def execute_ingest(args: argparse.Namespace, parser: argparse.ArgumentParser) -> parser: Parser used for the mutual-exclusion error. Returns: - int: 0, 1, 2, or 3. + int: 0, 1, 2, 3, or 4. """ if bool(args.source) == bool(args.pending): parser.error("provide a source or --pending, not both") @@ -415,9 +447,12 @@ def execute_ingest(args: argparse.Namespace, parser: argparse.ArgumentParser) -> raw_only=args.raw_only, force=args.force, dry_run=args.dry_run, + origin=args.origin, ) except IngestError as exc: print(f"Error: {exc}", file=sys.stderr) + if isinstance(exc, AuthRequiredError): + print(_AUTH_NEXT.format(url=exc.url), file=sys.stderr) return exc.exit_code if result.error: print(f"Error: {result.error}", file=sys.stderr) @@ -430,6 +465,20 @@ def execute_ingest(args: argparse.Namespace, parser: argparse.ArgumentParser) -> return result.exit_code +def _check_origin(argument: str, scheme: str | None, origin: str | None) -> None: + """Reject ``--origin`` with a URL source or a non-http value, before fetch. + + Generated by AI/LLM: 2026-10-07 - SPEC 7.1.5.2.9.1. + """ + if origin is None: + return + if scheme is not None: + raise IngestError(f"--origin needs a local file: {argument}") + origin_scheme = url_scheme(origin) + if origin_scheme is None or origin_scheme.lower() not in {"http", "https"}: + raise IngestError(f"--origin must be an http or https URL: {origin}") + + def _require_wiki(directory: str) -> pathlib.Path: """Return the wiki root or raise when settings are missing. @@ -461,8 +510,9 @@ def _fetched_line(source: FetchedSource) -> list[str]: """Return the ``Fetched:`` line for a URL, or an empty list. Generated by AI/LLM: 2026-09-30 - local files do not print Fetched. + Modified by AI/LLM: 2026-10-07 - a local file with ``--origin`` was not fetched either. """ - if source.origin_kind != "url": + if url_scheme(source.argument) is None: return [] return [f"Fetched: {source.argument} ({source.media_type}, {source.size_bytes} bytes)"] diff --git a/src/fiona/wiki/ingest/settings.py b/src/fiona/wiki/ingest/settings.py index 7e7538e..8a2b5b1 100644 --- a/src/fiona/wiki/ingest/settings.py +++ b/src/fiona/wiki/ingest/settings.py @@ -1,6 +1,7 @@ """Ingest settings from ``fiona-settings.json``. Generated by AI/LLM: 2026-09-30 - SPEC 5.1 ``fiona.wiki.ingest`` keys. +Modified by AI/LLM: 2026-10-07 - ``login_url_patterns`` and ``html.*`` keys. """ from __future__ import annotations @@ -8,7 +9,18 @@ from __future__ import annotations import dataclasses import importlib.metadata import json +import logging import pathlib +import re + +logger = logging.getLogger(__name__) + +DEFAULT_LOGIN_URL_PATTERNS = ( + r"/(login|logon|signin|sign-in)\b", + r"/(sso|saml2?|oauth2?|adfs|cas)(/|$)", + r"login\.microsoftonline\.com", + r"j_security_check", +) def package_version() -> str: @@ -30,6 +42,7 @@ class IngestSettings: """Runtime limits and paths for one ingest. Generated by AI/LLM: 2026-09-30 - defaults from the settings table. + Modified by AI/LLM: 2026-10-07 - login patterns and block sweep limits. """ timeout_seconds: float @@ -37,6 +50,11 @@ class IngestSettings: user_agent: str default_target: str raw_categories: dict[str, str] + login_url_patterns: list[str] = dataclasses.field( + default_factory=lambda: list(DEFAULT_LOGIN_URL_PATTERNS) + ) + html_min_words: int = 4 + html_max_link_density: float = 0.5 def default_settings() -> IngestSettings: @@ -60,6 +78,7 @@ def load_ingest_settings(wiki_root: pathlib.Path) -> IngestSettings: """Load ingest settings, falling back per key when a value is unusable. Generated by AI/LLM: 2026-09-30 - absent or mistyped keys keep the default. + Modified by AI/LLM: 2026-10-07 - read ``login_url_patterns`` and ``html``. Args: wiki_root: Directory that contains ``fiona-settings.json``. @@ -104,4 +123,31 @@ def load_ingest_settings(wiki_root: pathlib.Path) -> IngestSettings: if isinstance(value, str) and value.strip() } settings.raw_categories = cleaned + patterns = section.get("login_url_patterns") + if isinstance(patterns, list): + settings.login_url_patterns = _compilable(patterns) + html = section.get("html") + if isinstance(html, dict): + words = html.get("min_words") + if isinstance(words, int) and not isinstance(words, bool) and words >= 1: + settings.html_min_words = words + density = html.get("max_link_density") + if isinstance(density, (int, float)) and not isinstance(density, bool) and 0 <= density <= 1: + settings.html_max_link_density = float(density) return settings + + +def _compilable(patterns: list) -> list[str]: + """Return the entries that compile as regular expressions. + + Generated by AI/LLM: 2026-10-07 - SPEC 5.1, one warning per skipped entry. + """ + kept: list[str] = [] + for pattern in patterns: + try: + re.compile(pattern) + except (re.error, TypeError): + logger.warning("login_url_patterns: skipped %r; not a valid regular expression", pattern) + continue + kept.append(pattern) + return kept diff --git a/src/fiona/wiki/templates/graph_physics.js b/src/fiona/wiki/templates/graph_physics.js index c5005bc..444b8a5 100644 --- a/src/fiona/wiki/templates/graph_physics.js +++ b/src/fiona/wiki/templates/graph_physics.js @@ -214,8 +214,17 @@ return `rgb(${r}, ${g}, ${b})`; } + function isOrphan(d) { + return d.kind !== 'external' && d.kind !== 'source' + && d.in_degree === 0 && d.out_degree === 0; + } + // Color follows link_score. Repeats from one file decay by half. + // AI/LLM: 2026-10-07 - a theme can paint orphans one color (SPEC 5.3.5). function getNodeColor(d) { + if (theme.orphan && theme.orphan.enabled && isOrphan(d)) { + return theme.orphan.node_color; + } const low = theme.node_colors.low; const mid = theme.node_colors.mid; const high = theme.node_colors.high; @@ -1241,9 +1250,7 @@ if (d.x == null || d.y == null) return false; if (d.kind === 'external' && !showExternalLinks) return false; if (d.kind === 'source' && !showSources) return false; - const orphan = d.kind !== 'external' && d.kind !== 'source' - && d.in_degree === 0 && d.out_degree === 0; - if (orphan && !showOrphans) return false; + if (isOrphan(d) && !showOrphans) return false; return true; }); } diff --git a/src/fiona/wiki/update_manager.py b/src/fiona/wiki/update_manager.py index 3621280..36aa120 100644 --- a/src/fiona/wiki/update_manager.py +++ b/src/fiona/wiki/update_manager.py @@ -60,6 +60,9 @@ SETTINGS_FILE = "fiona-settings.json" # Generated by AI/LLM: 2026-10-02 - record written only by wiki update (SPEC 7.1.2.14) COREFILES_NAME = "corefiles.json" +# AI/LLM: 2026-10-14 - added when missing, never compared after that (SPEC 3.4.4) +MEMORY_NAME = "MEMORY.md" + logger = logging.getLogger(__name__) # Read-only zones that should never be modified @@ -75,6 +78,7 @@ __all__ = [ "COREFILES_NAME", "CORE_DIRECTORIES", "CORE_FILES", + "MEMORY_NAME", "READ_ONLY_ZONES", "SETTINGS_FILE", "UPDATE_SUFFIX", @@ -720,11 +724,13 @@ def _considered_core_file( ) -> bool: """Return whether update compares this template-relative path. - Modified by AI/LLM: 2026-10-02 - skip fiona.wiki.update.ignore_files. + Modified by AI/LLM: 2026-10-14 - skip ignore_files and MEMORY.md (SPEC 3.4.4). """ wiki_file = wiki_root / relative if path_is_update_ignored(relative.as_posix(), ignore_patterns): return False + if relative.as_posix() == MEMORY_NAME: + return False if _is_protected(relative, wiki_root, patterns): return False wiki_page = _is_wiki_page_path(relative) @@ -829,6 +835,9 @@ def check_wiki_updates(directory: str | Path = ".") -> UpdateSummary: if not wiki_file.exists(): summary.files_to_add.append(relative) continue + # AI/LLM: 2026-10-14 - the wiki's memory evolves with the wiki (SPEC 3.4.4) + if relative.as_posix() == MEMORY_NAME: + continue result = _compare_core_file(wiki_file, template_file, relative) identity = _file_identity(result, template_file) diff --git a/tests/test_bootstrap_install.py b/tests/test_bootstrap_install.py index f003ec2..d10a8ec 100644 --- a/tests/test_bootstrap_install.py +++ b/tests/test_bootstrap_install.py @@ -5,10 +5,18 @@ Generated by AI/LLM: 2026-09-30 - local and no-env install options. import importlib.util import pathlib +import shutil import subprocess import pytest +# AI/LLM: Linux CI images (GitLab python:) have no PowerShell. +POWERSHELL = shutil.which("powershell") or shutil.which("pwsh") +requires_powershell = pytest.mark.skipif( + not POWERSHELL, + reason="PowerShell not installed (Windows-only bootstrap.ps1 tests)", +) + ROOT = pathlib.Path(__file__).resolve().parents[1] @@ -36,9 +44,10 @@ def _script(name: str) -> str: def _powershell(*args: str) -> subprocess.CompletedProcess[str]: """Run bootstrap.ps1 as a file, not dot-sourced.""" + assert POWERSHELL is not None return subprocess.run( [ - "powershell", + POWERSHELL, "-NoProfile", "-ExecutionPolicy", "Bypass", @@ -138,6 +147,7 @@ class TestBootstrapScripts: """Bash does not create a virtual environment when NO_ENV is true.""" assert '[ "$NO_ENV" = true ]' in _script("bootstrap") + @requires_powershell def test_powershell_local_without_no_env_must_be_dot_sourced(self): """--local alone is a venv install and must be dot-sourced.""" result = _powershell("--local") @@ -145,6 +155,7 @@ class TestBootstrapScripts: assert "Dot-source" in combined assert "Unknown option" not in combined + @requires_powershell def test_powershell_rejects_unknown_option(self): """An unrecognized flag is still reported as unknown.""" result = _powershell("--not-a-flag") diff --git a/tests/test_gittool.py b/tests/test_gittool.py index 2d5cf44..e4968d1 100644 --- a/tests/test_gittool.py +++ b/tests/test_gittool.py @@ -8,6 +8,16 @@ import pytest from fiona.gittool.git_tool import GitTool +def _configure_test_repo(repo: git.Repo) -> None: + """Set local git identity so commits and tags work without global config. + + Generated by AI/LLM: 2026-10-14 - GitLab CI runs as root without user.name. + """ + with repo.config_writer() as config: + config.set_value("user", "email", "fiona-test@example.com") + config.set_value("user", "name", "Fiona Test") + + @pytest.fixture(scope="session") def read_only_repo(tmp_path_factory): """One git repo for tests that only read it. @@ -16,6 +26,7 @@ def read_only_repo(tmp_path_factory): """ path = tmp_path_factory.mktemp("read_only_git") repo = git.Repo.init(path) + _configure_test_repo(repo) (path / "file1.txt").write_text("initial content") repo.index.add(["file1.txt"]) repo.index.commit("Initial commit") @@ -26,6 +37,7 @@ def read_only_repo(tmp_path_factory): def git_repo(tmp_path): """Create a temporary git repository for testing.""" repo = git.Repo.init(tmp_path) + _configure_test_repo(repo) # Create an initial commit (tmp_path / "file1.txt").write_text("initial content") repo.index.add(["file1.txt"]) @@ -297,3 +309,20 @@ class TestGitTool: assert repo is not None commits = list(repo.iter_commits()) assert len(commits) >= 1 + + def test_init_wiki_repo_ignores_obsidian_user_state(self, tmp_path): + """SPEC 3.2.3: Obsidian per-user state is ignored, vault settings are not. + + Generated by AI/LLM: 2026-10-15 - Obsidian vault support. + """ + repo = GitTool.init_wiki_repo(tmp_path, "Initial commit") + + paths = [ + ".obsidian/workspace.json", + ".obsidian/workspace-mobile.json", + ".obsidian/cache", + ".obsidian/app.json", + ] + ignored = repo.git.check_ignore(*paths).splitlines() + + assert ignored == paths[:3] diff --git a/tests/test_graph_generator.py b/tests/test_graph_generator.py index 6e39af7..de494ef 100644 --- a/tests/test_graph_generator.py +++ b/tests/test_graph_generator.py @@ -349,6 +349,43 @@ class TestConstellationThemes: assert "broken" not in themes assert "colorful" in themes + def test_packaged_themes_have_an_orphan_color_turned_off(self): + """SPEC 5.3.5: every packaged theme ships orphan coloring off. + + Generated by AI/LLM: 2026-10-07 - orphan color per theme. + """ + themes = load_constellation_themes() + for name, theme in themes.items(): + assert theme["orphan"]["enabled"] is False, name + assert theme["orphan"]["node_color"].startswith("#"), name + + def test_local_theme_turns_orphan_color_on(self, tmp_path): + """SPEC 5.3.5: enabling orphans keeps the packaged orphan color. + + Generated by AI/LLM: 2026-10-07 - orphan.enabled overlays one key. + """ + theme_dir = tmp_path / "constellation" + theme_dir.mkdir() + (theme_dir / "dark.json").write_text( + json.dumps({"orphan": {"enabled": True}}), + encoding="utf-8", + ) + packaged = load_constellation_themes()["dark"]["orphan"]["node_color"] + orphan = load_constellation_themes(str(tmp_path))["dark"]["orphan"] + assert orphan["enabled"] is True + assert orphan["node_color"] == packaged + + def test_html_reads_the_orphan_theme_keys(self): + """SPEC 5.3.5: the page script paints orphans from the theme. + + Generated by AI/LLM: 2026-10-07 - orphan color in the graph HTML. + """ + graph = WikiGraph() + graph.add_node("wiki/file1.md") + html = HTMLGraphRenderer(graph).generate_html() + assert "theme.orphan.enabled" in html + assert "theme.orphan.node_color" in html + def test_node_radius_keeps_a_fraction(self, tmp_path): """A fractional min and max stay fractional and drive the drawing. @@ -1701,25 +1738,28 @@ class TestWikiGraphMain: ) assert result == 1 - def test_main_output_not_writable(self, tmp_path): + def test_main_output_not_writable(self, tmp_path, monkeypatch): """Test main function when output file is not writable (lines 687-690).""" md_file1 = tmp_path / "file1.md" md_file1.write_text("# File 1\n") - # Create a read-only output file output_file = tmp_path / "graph.html" output_file.write_text("existing content") - os.chmod(output_file, 0o444) # Read-only + target = os.path.normcase(os.path.abspath(output_file)) + real_access = os.access - try: - result = wiki_graph_main( - directory=str(tmp_path), - output=str(output_file) - ) - assert result == 1 - finally: - # Restore permissions for cleanup - os.chmod(output_file, 0o644) + def deny_file_write(path, mode): + if os.path.normcase(os.path.abspath(path)) == target and mode == os.W_OK: + return False + return real_access(path, mode) + + monkeypatch.setattr("os.access", deny_file_write) + + result = wiki_graph_main( + directory=str(tmp_path), + output=str(output_file) + ) + assert result == 1 def test_main_output_directory_not_writable(self, tmp_path, monkeypatch): """Test main function when output directory is not writable (lines 694-696).""" @@ -1889,10 +1929,12 @@ class TestGraphIgnoreFiles: ) def test_shipped_settings_ignore_table_of_contents(self): - """SPEC 7.1.4.20: the template lists the TOC glob.""" + """SPEC 7.1.4.20: the template lists the TOC, README, and AGENTS globs.""" settings = json.loads( (llm_wiki_template() / "fiona-settings.json").read_text(encoding="utf-8") ) assert settings["fiona"]["wiki"]["graph"]["ignore_files"] == [ - "*/table_of_contents.md" + "*/table_of_contents.md", + "*/README.md", + "*/AGENTS.md", ] diff --git a/tests/test_html_sweep.py b/tests/test_html_sweep.py new file mode 100644 index 0000000..e7ab28a --- /dev/null +++ b/tests/test_html_sweep.py @@ -0,0 +1,399 @@ +"""Tests for the HTML block sweep (SPEC 7.1.5.4.5.6) and its settings (SPEC 5.1). + +Generated by AI/LLM: 2026-10-07 - one test per acceptance of 7.1.5.4.5.6. +""" + +from __future__ import annotations + +import json +import pathlib +import re +import shutil + +import pytest + +from fiona.paths import llm_wiki_template +from fiona.wiki.ingest.converters import default_registry +from fiona.wiki.ingest.converters.html import HtmlConverter +from fiona.wiki.ingest.models import ConversionResult, FetchedSource +from fiona.wiki.ingest.pipeline import default_pipeline +from fiona.wiki.ingest.settings import default_settings, load_ingest_settings +from fiona.wiki.sanity_manager import run_wiki_checks + +_REVIEW = re.compile(r"<!-- fiona:review (\w+) -->\n\n(.*?)\n\n<!-- /fiona:review -->", re.DOTALL) + +_LANDING = """<!doctype html> +<html><head><title>Agent harness +
+

How it works

+
Run every agent against the same suite of tests and compare results
+
+
The harness cut our regression time in half overnight.
+

Ada, Example Corp

+
We ship agents with far more confidence than before.
+

Bob, Other Inc

+ +""" + +_BOILERPLATE = """ + +
+

Twelve words of real content sit in this one paragraph right here.

+\u2022 +
Share
+
Share
+

Twelve words of real content sit in this one paragraph right here.

+ +
Tiny note
+
+

Copyright notice for this example site goes here.

+ +""" + +_ENCYCLOPEDIA = """ + +
+

Ada Lovelace

+ +
Born10 December 1815
Known forNotes on the Analytical Engine
+

Ada Lovelace was an English mathematician and writer.[1]

+

She is known for her work on the Analytical Engine of Charles Babbage.[2]

+

References

+
  1. Toole, Ada: The Enchantress of Numbers
  2. +
  3. Essinger, Ada's Algorithm
+ + +
+""" + +_QA = """ +
42
+

How do I print every item of a list in reverse order?

+
7
+

Loop over the reversed view so nothing is copied.

+
for item in reversed(items): print(item)
+
3
+

The same loop also works on any sequence type.

+
for item in reversed(items): print(item)
+ + +""" + +_FORUM = """
+First post body text written by the original author +Reply number one adds a useful detail +HIDDENREPLY should never appear in output + + +Third post closes the thread with thanks +
+""" + +_LAYOUT_TABLE = """ + + +
Post one has enough words to keep
Post two also has enough words here
+""" + + +def _convert(tmp_path: pathlib.Path, html: str, **options: float) -> ConversionResult: + """Convert one HTML string with the given sweep options.""" + path = tmp_path / "page.html" + path.write_text(html, encoding="utf-8") + source = FetchedSource( + argument=str(path), + path=path, + origin=str(path), + origin_kind="file", + final_url=None, + media_type="text/html", + charset=None, + size_bytes=path.stat().st_size, + slug="page", + fetched="2026-10-07T00:00:00Z", + created=False, + ) + return HtmlConverter(**options).convert(source) + + +def _reviews(body: str) -> list[tuple[str, str]]: + """Return ``(reason, markdown)`` for every review block.""" + return _REVIEW.findall(body) + + +def _outside(body: str) -> str: + """Return the body with every review block removed.""" + return _REVIEW.sub("", body) + + +def _wiki(tmp_path: pathlib.Path, html_settings: dict | None = None) -> pathlib.Path: + """Create a small llm-wiki, optionally with ``fiona.wiki.ingest.html``.""" + root = tmp_path / "wiki" + (root / "wiki").mkdir(parents=True) + (root / "ingest").mkdir() + (root / "raw").mkdir() + shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json") + if html_settings is not None: + settings = json.loads((root / "fiona-settings.json").read_text(encoding="utf-8")) + settings.setdefault("fiona", {}).setdefault("wiki", {})["ingest"] = {"html": html_settings} + (root / "fiona-settings.json").write_text(json.dumps(settings), encoding="utf-8") + (root / "wiki" / "log.md").write_text( + "---\nname: log\nversion: 2.0.0\nupdated: 2026-09-30\n---\n\n# Log\n", + encoding="utf-8", + newline="\n", + ) + return root + + +def test_landing_page_keeps_every_section(tmp_path: pathlib.Path) -> None: + """A section's heading and div, then both cards, all in order.""" + body = _convert(tmp_path, _LANDING).body + outside = _outside(body) + order = [ + "## How it works", + "Run every agent against the same suite of tests and compare results", + "The harness cut our regression time in half overnight.", + "We ship agents with far more confidence than before.", + ] + positions = [outside.index(text) for text in order] + assert positions == sorted(positions) + + +def test_boilerplate_verdicts_and_reasons(tmp_path: pathlib.Path) -> None: + """Nav and the bullet are dropped; chrome, repeat, links, and short are marked.""" + result = _convert(tmp_path, _BOILERPLATE) + body = result.body + assert "Home" not in body + assert "Docs" not in body + assert "\u2022" not in body + assert body.count("Share") == 1 + paragraph = "Twelve words of real content sit in this one paragraph right here." + assert paragraph in _outside(body) + reasons = dict((text.strip(), reason) for reason, text in _reviews(body)) + assert reasons["Copyright notice for this example site goes here."] == "chrome" + assert reasons[paragraph] == "repeat" + assert reasons["Tiny note"] == "short" + assert reasons["Share"] == "short" + links = [text for text, reason in reasons.items() if reason == "links"] + assert len(links) == 1 + assert "Alpha link" in links[0] + assert "html sweep: 1 keep, 5 review, 3 drop" in result.warnings + assert result.conversion == "complete" + + +def test_encyclopedia_page(tmp_path: pathlib.Path) -> None: + """Body paragraphs keep; references and navbox are links; no menu text.""" + body = _convert(tmp_path, _ENCYCLOPEDIA).body + outside = _outside(body) + assert "Ada Lovelace was an English mathematician and writer." in outside + assert "She is known for her work on the Analytical Engine of Charles Babbage." in outside + assert "Main page" not in body + assert "Random article" not in body + reasons = [(reason, text) for reason, text in _reviews(body)] + assert any(reason == "links" and "Enchantress" in text for reason, text in reasons) + assert any(reason == "links" and "Boole" in text for reason, text in reasons) + + +def test_qa_page(tmp_path: pathlib.Path) -> None: + """Question and answers keep in order; repeated pre, vote counts, and aside are marked.""" + body = _convert(tmp_path, _QA).body + outside = _outside(body) + order = [ + "How do I print every item of a list in reverse order?", + "Loop over the reversed view so nothing is copied.", + "The same loop also works on any sequence type.", + ] + positions = [outside.index(text) for text in order] + assert positions == sorted(positions) + assert outside.count("for item in reversed(items)") == 1 + reasons = _reviews(body) + assert [reason for reason, text in reasons if "reversed(items)" in text] == ["repeat"] + assert sorted(text.strip() for reason, text in reasons if reason == "short") == ["3", "42", "7"] + assert {reason for reason, text in reasons if "Related questions" in text or "Sort a list" in text} == { + "chrome" + } + + +def test_forum_thread(tmp_path: pathlib.Path) -> None: + """Custom-element posts, a shadow-root post, and replies in order; hidden reply gone.""" + body = _convert(tmp_path, _FORUM).body + order = [ + "First post body text written by the original author", + "Reply number one adds a useful detail", + "Shadow post text that lives in a template", + "Third post closes the thread with thanks", + ] + positions = [_outside(body).index(text) for text in order] + assert positions == sorted(positions) + assert "HIDDENREPLY" not in body + + +def test_layout_table_gives_one_block_per_post(tmp_path: pathlib.Path) -> None: + """Cells that hold divs are containers, not a pipe table.""" + body = _convert(tmp_path, _LAYOUT_TABLE).body + assert "|" not in body + assert "Post one has enough words to keep\n\nPost two also has enough words here" in body + + +def test_data_table_stays_a_pipe_table(tmp_path: pathlib.Path) -> None: + """A table whose cells hold only text and inline tags is one block.""" + html = "
Col one
Cell value
" + assert "| Col one |" in _convert(tmp_path, html).body + + +def test_sweep_is_deterministic(tmp_path: pathlib.Path) -> None: + """Two conversions of one fixture give the same body and counts.""" + first = _convert(tmp_path, _BOILERPLATE) + second = _convert(tmp_path, _BOILERPLATE) + assert first.body == second.body + assert first.warnings == second.warnings + + +def test_hidden_markers_drop(tmp_path: pathlib.Path) -> None: + """``hidden``, ``aria-hidden``, ``visibility: hidden``, button, and select text is dropped.""" + html = """ + +

SECRETB one two three four

+

SECRETC one two three four

+ + +

Visible text with enough words here

+""" + body = _convert(tmp_path, html).body + assert "SECRET" not in body + assert "Visible text with enough words here" in body + + +def test_chrome_roles_and_header(tmp_path: pathlib.Path) -> None: + """Roles and a top-level header are chrome; a header inside article is not.""" + html = """ +

Site banner text with many words

+

Side widget text with many words

+

Form intro text with many words

+

Article byline with many words

+""" + body = _convert(tmp_path, html).body + reasons = dict((text.strip(), reason) for reason, text in _reviews(body)) + assert reasons["Site banner text with many words"] == "chrome" + assert reasons["Side widget text with many words"] == "chrome" + assert reasons["Form intro text with many words"] == "chrome" + assert "Article byline with many words" in _outside(body) + + +def test_text_folding_and_word_count(tmp_path: pathlib.Path) -> None: + """``Fast`` is one word and ``
`` separates words.""" + joined = _convert(tmp_path, "
Fast go now
").body + assert _reviews(joined) == [("short", "**Fa**st go now")] + split = _convert(tmp_path, "

one
two three four

").body + assert _reviews(split) == [] + + +def test_inline_card_holding_a_block_is_a_container(tmp_path: pathlib.Path) -> None: + """``
...
`` is walked, so the card text is not one link run.""" + html = '
Card text with enough words inside it
' + body = _convert(tmp_path, html).body + assert "Card text with enough words inside it" in _outside(body) + + +def test_every_block_in_review_is_partial(tmp_path: pathlib.Path) -> None: + """No keep and at least one review is partial with the second warning.""" + result = _convert(tmp_path, "

Tiny

") + assert result.conversion == "partial" + assert "no block passed the content checks; every block is marked for review" in result.warnings + + +def test_empty_body_rule_runs_first(tmp_path: pathlib.Path) -> None: + """A page with no text keeps the 7.1.5.4.5.5 warning and counts zero blocks.""" + result = _convert(tmp_path, " ") + assert result.conversion == "partial" + assert result.warnings[0].startswith("no text content") + assert "html sweep: 0 keep, 0 review, 0 drop" in result.warnings + + +def test_html_without_body_skips_head(tmp_path: pathlib.Path) -> None: + """A fragment without ```` is swept without the head's title text.""" + html = "Only Title

Fragment text with enough words

" + result = _convert(tmp_path, html) + assert result.title == "Only Title" + assert result.body == "Fragment text with enough words\n" + + +def test_min_words_option(tmp_path: pathlib.Path) -> None: + """``min_words`` decides when a loose block is short.""" + html = "
Three word text
" + assert _reviews(_convert(tmp_path, html).body) == [("short", "Three word text")] + assert _reviews(_convert(tmp_path, html, min_words=2).body) == [] + + +def test_settings_defaults_and_overrides(tmp_path: pathlib.Path) -> None: + """SPEC 5.1: ``html.min_words`` and ``html.max_link_density`` validate their values.""" + defaults = default_settings() + assert (defaults.html_min_words, defaults.html_max_link_density) == (4, 0.5) + good = load_ingest_settings(_wiki(tmp_path / "good", {"min_words": 6, "max_link_density": 1})) + assert (good.html_min_words, good.html_max_link_density) == (6, 1.0) + for index, bad in enumerate((0, True, "4", 1.5)): + loaded = load_ingest_settings(_wiki(tmp_path / f"words{index}", {"min_words": bad})) + assert loaded.html_min_words == 4, bad + for index, bad in enumerate((True, "0.5", 1.5, -0.1)): + loaded = load_ingest_settings(_wiki(tmp_path / f"density{index}", {"max_link_density": bad})) + assert loaded.html_max_link_density == 0.5, bad + zero = load_ingest_settings(_wiki(tmp_path / "zero", {"max_link_density": 0})) + assert zero.html_max_link_density == 0.0 + odd = _wiki(tmp_path / "odd") + settings = json.loads((odd / "fiona-settings.json").read_text(encoding="utf-8")) + settings.setdefault("fiona", {}).setdefault("wiki", {})["ingest"] = {"html": "nope"} + (odd / "fiona-settings.json").write_text(json.dumps(settings), encoding="utf-8") + assert load_ingest_settings(odd).html_min_words == 4 + + +def test_registry_hands_settings_to_html(tmp_path: pathlib.Path) -> None: + """``default_registry(settings)`` builds the html converter with those values.""" + settings = default_settings() + settings.html_min_words = 9 + converter = default_registry(settings).find("text/html", ".html") + assert isinstance(converter, HtmlConverter) + assert converter.min_words == 9 + assert default_registry().find("text/html", "").min_words == 4 + + +def test_pipeline_uses_wiki_min_words(tmp_path: pathlib.Path) -> None: + """A wiki with ``min_words: 2`` keeps a three-word div through the full pipeline.""" + root = _wiki(tmp_path, {"min_words": 2}) + source = tmp_path / "three.html" + source.write_text("
Three word text
", encoding="utf-8") + result = default_pipeline(root).run(str(source)) + assert result.exit_code == 0 + page = (root / result.page_path).read_text(encoding="utf-8") + assert "Three word text" in page + assert "fiona:review" not in page + + +def test_review_markers_pass_wiki_sanity(tmp_path: pathlib.Path) -> None: + """An ingested page with review blocks keeps its markers and passes markdown lint.""" + root = _wiki(tmp_path) + source = tmp_path / "sweep-page.html" + source.write_text(_BOILERPLATE, encoding="utf-8") + result = default_pipeline(root).run(str(source)) + assert result.exit_code == 0 + page = (root / result.page_path).read_text(encoding="utf-8") + assert len(_reviews(page)) == 5 + assert not any("markdown style violations" in warning for warning in result.warnings) + results = run_wiki_checks(root) + hits = [ + violation + for check in results["checks"].values() + for violation in check.get("violations", []) + if "sweep-page.md" in str(violation) + ] + assert [hit for hit in hits if "MD" in str(hit)] == [] + + +@pytest.mark.parametrize("fixture", [_LANDING, _BOILERPLATE, _ENCYCLOPEDIA, _QA, _FORUM, _LAYOUT_TABLE]) +def test_markers_sit_on_their_own_lines(tmp_path: pathlib.Path, fixture: str) -> None: + """Every marker is a whole line and opens and closes in pairs.""" + lines = _convert(tmp_path, fixture).body.splitlines() + opens = [line for line in lines if "fiona:review" in line and "/fiona:review" not in line] + closes = [line for line in lines if "/fiona:review" in line] + assert all(re.fullmatch(r"", line) for line in opens) + assert all(line == "" for line in closes) + assert len(opens) == len(closes) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 3cf0d02..303b984 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -498,6 +498,26 @@ def test_ingest_drop_moves_and_personal_is_copied(tmp_path: pathlib.Path) -> Non assert (root / "raw" / "articles" / "note.txt").read_bytes() == original +def test_origin_inside_wiki_is_relative(tmp_path: pathlib.Path) -> None: + """SPEC 7.1.5.2.10: a file inside the wiki is recorded relative to the wiki root.""" + root = _wiki(tmp_path) + dropped = root / "ingest" / "sub" / "a.txt" + dropped.parent.mkdir() + dropped.write_text("inside\n", encoding="utf-8", newline="\n") + code, stdout, stderr = _run(root, str(dropped.resolve()), "--json") + assert code == 0, stderr + assert json.loads(stdout)["origin"] == "ingest/sub/a.txt" + front = _front(root / "wiki" / "sources" / "a.md") + assert front["ingest"]["origin"] == "ingest/sub/a.txt" + assert front["ingest"]["origin_kind"] == "file" + assert "ingest ingest/sub/a.txt -> " in (root / "wiki" / "log.md").read_text(encoding="utf-8") + outside = tmp_path / "b.txt" + outside.write_text("outside\n", encoding="utf-8", newline="\n") + code, stdout, stderr = _run(root, str(outside), "--json") + assert code == 0, stderr + assert json.loads(stdout)["origin"] == str(outside.resolve()) + + def test_unknown_bytes_archive_without_a_page(tmp_path: pathlib.Path) -> None: """SPEC 7.1.5.4.1: random bytes exit 3 under ``raw/other``.""" root = _wiki(tmp_path) diff --git a/tests/test_ingest_auth.py b/tests/test_ingest_auth.py new file mode 100644 index 0000000..99e6813 --- /dev/null +++ b/tests/test_ingest_auth.py @@ -0,0 +1,350 @@ +"""Tests for login detection, exit 4, and ``--origin`` (SPEC 7.1.5.2.8, 7.1.5.2.9). + +Generated by AI/LLM: 2026-10-07 - fake openers only; no test makes a network call. +""" + +from __future__ import annotations + +import contextlib +import email.message +import hashlib +import io +import json +import pathlib +import shutil +import typing +import urllib.error +import urllib.request + +import pytest +import yaml + +from fiona.paths import llm_wiki_template +from fiona.wiki.ingest import pipeline as pipeline_mod +from fiona.wiki.ingest.converters import default_registry +from fiona.wiki.ingest.fetchers import UrlFetcher +from fiona.wiki.ingest.models import AuthRequiredError, IngestError +from fiona.wiki.ingest.pipeline import default_pipeline, ingest_main +from fiona.wiki.ingest.settings import DEFAULT_LOGIN_URL_PATTERNS, load_ingest_settings + +_NEXT = "Next: save it into ingest/, then run: wiki ingest --origin " + + +class _Response: + """One canned 200 response.""" + + def __init__(self, body: bytes, header: str, url: str) -> None: + self._body = body + self._offset = 0 + self.headers = email.message.Message() + self.headers["Content-Type"] = header + self._url = url + + def read(self, amount: int = -1) -> bytes: + """Return the next chunk of the body.""" + if amount < 0: + amount = len(self._body) - self._offset + chunk = self._body[self._offset:self._offset + amount] + self._offset += len(chunk) + return chunk + + def geturl(self) -> str: + """Return the URL after redirects.""" + return self._url + + def __enter__(self) -> typing.Self: + return self + + def __exit__(self, *_args: object) -> bool: + return False + + +class _Opener: + """Opener that answers one status, or 200 with a body at ``final_url``.""" + + def __init__( + self, + body: bytes = b"", + header: str = "text/html", + *, + status: int = 200, + final_url: str | None = None, + ) -> None: + self.body = body + self.header = header + self.status = status + self.final_url = final_url + self.calls = 0 + + def open(self, request: urllib.request.Request, timeout: float | None = None) -> _Response: + """Return the canned response or raise the canned HTTP error.""" + del timeout + self.calls += 1 + if self.status != 200: + raise urllib.error.HTTPError( + request.full_url, + self.status, + "server phrase", + email.message.Message(), + io.BytesIO(b""), + ) + return _Response(self.body, self.header, self.final_url or request.full_url) + + +def _wiki(tmp_path: pathlib.Path, ingest: dict | None = None) -> pathlib.Path: + """Create a small llm-wiki, optionally with ``fiona.wiki.ingest`` settings.""" + root = tmp_path / "wiki" + (root / "wiki").mkdir(parents=True) + (root / "ingest").mkdir() + (root / "raw").mkdir() + shutil.copy2(llm_wiki_template() / "fiona-settings.json", root / "fiona-settings.json") + if ingest is not None: + settings = json.loads((root / "fiona-settings.json").read_text(encoding="utf-8")) + settings.setdefault("fiona", {}).setdefault("wiki", {})["ingest"] = ingest + (root / "fiona-settings.json").write_text(json.dumps(settings), encoding="utf-8") + (root / "wiki" / "log.md").write_text( + "---\nname: log\nversion: 2.0.0\nupdated: 2026-09-30\n---\n\n# Log\n", + encoding="utf-8", + newline="\n", + ) + return root + + +def _hashes(root: pathlib.Path) -> dict[str, str]: + """Hash every file under a wiki.""" + return { + path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in root.rglob("*") + if path.is_file() + } + + +def _cli( + root: pathlib.Path, + opener: _Opener | None, + monkeypatch: pytest.MonkeyPatch, + *args: str, +) -> tuple[int, str, str]: + """Run the ingest CLI with the URL fetcher bound to ``opener``.""" + if opener is not None: + original = pipeline_mod.default_pipeline + + def patched(wiki_root: pathlib.Path) -> pipeline_mod.IngestPipeline: + built = original(wiki_root) + built.fetchers[0] = UrlFetcher(built.registry, opener=opener) + return built + + monkeypatch.setattr(pipeline_mod, "default_pipeline", patched) + stdout = io.StringIO() + stderr = io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = ingest_main(["--wiki", str(root), *args]) + return code, stdout.getvalue(), stderr.getvalue() + + +def _front(path: pathlib.Path) -> dict: + """Parse page frontmatter.""" + return yaml.safe_load(path.read_text(encoding="utf-8").split("---", 2)[1]) + + +@pytest.mark.parametrize("status, phrase", [(401, "401 Unauthorized"), (403, "403 Forbidden")]) +def test_status_401_403_exit_4_and_write_nothing( + tmp_path: pathlib.Path, + monkeypatch: pytest.MonkeyPatch, + status: int, + phrase: str, +) -> None: + """SPEC 7.1.5.2.8: 401 and 403 exit 4 with both stderr lines and no file change.""" + root = _wiki(tmp_path) + before = _hashes(root) + code, stdout, stderr = _cli(root, _Opener(status=status), monkeypatch, "https://host/job/x/console") + assert code == 4 + assert stdout == "" + lines = stderr.strip().splitlines() + assert lines == [ + f"Error: Authentication required: https://host/job/x/console: {phrase}", + _NEXT + "https://host/job/x/console", + ] + assert "Traceback" not in stderr + assert _hashes(root) == before + + +def test_redirect_to_login_exits_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """SPEC 7.1.5.2.8.2: a redirect whose host and path match a pattern is a login.""" + root = _wiki(tmp_path) + before = _hashes(root) + final = "https://host/login?from=%2Fjob%2Fx%2Fconsole" + opener = _Opener(b"

Sign in to continue please

", final_url=final) + code, _stdout, stderr = _cli(root, opener, monkeypatch, "https://host/job/x/console") + assert code == 4 + assert f"redirected to {final}" in stderr + assert _hashes(root) == before + + +def test_login_form_exits_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """SPEC 7.1.5.2.8.2: an HTML body with a password input is a login page.""" + root = _wiki(tmp_path) + before = _hashes(root) + body = b"
" + code, _stdout, stderr = _cli(root, _Opener(body), monkeypatch, "https://host/wiki/page") + assert code == 4 + assert "login page at https://host/wiki/page" in stderr + assert _hashes(root) == before + + +def test_login_form_unquoted_type() -> None: + """The password check accepts unquoted and double-quoted ``type`` values.""" + registry = default_registry() + for markup in (b'', b""): + fetcher = UrlFetcher(registry, opener=_Opener(b"" + markup + b"")) + with pytest.raises(AuthRequiredError, match="login page at"): + fetcher.fetch("https://host/p", pathlib.Path("unused")) + + +def test_docs_login_page_without_redirect_ingests(tmp_path: pathlib.Path) -> None: + """SPEC 7.1.5.2.8.3: a page whose own path holds ``login`` still ingests.""" + root = _wiki(tmp_path) + body = b"

How to configure the login screen for users

" + pipeline = default_pipeline(root) + pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body)) + result = pipeline.run("https://host/docs/login") + assert result.exit_code == 0 + assert result.page_path is not None + + +def test_redirect_elsewhere_ingests(tmp_path: pathlib.Path) -> None: + """A redirect to a path that matches no pattern is a normal download.""" + root = _wiki(tmp_path) + body = b"

Moved content that is still the real page

" + pipeline = default_pipeline(root) + pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://host/new")) + assert pipeline.run("https://host/old").exit_code == 0 + + +def test_404_still_download_failed(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """SPEC 7.1.5.2.3: other non-2xx statuses keep exit 1.""" + root = _wiki(tmp_path) + code, _stdout, stderr = _cli(root, _Opener(status=404), monkeypatch, "https://host/missing") + assert code == 1 + assert "Download failed" in stderr + assert "Next:" not in stderr + + +def test_json_and_dry_run_exit_4(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """SPEC 7.1.5.2.8.5: ``--json`` prints nothing on stdout; ``--dry-run`` exits 4.""" + root = _wiki(tmp_path) + before = _hashes(root) + opener = _Opener(status=401) + code, stdout, stderr = _cli(root, opener, monkeypatch, "https://host/a", "--json") + assert (code, stdout) == (4, "") + assert "Authentication required" in stderr + code, stdout, stderr = _cli(root, None, monkeypatch, "https://host/a", "--dry-run") + assert (code, stdout) == (4, "") + assert _hashes(root) == before + + +def test_userinfo_removed_from_both_lines(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """SPEC 7.1.5.2.8.4: the URL in both lines has no userinfo.""" + root = _wiki(tmp_path) + code, _stdout, stderr = _cli(root, _Opener(status=401), monkeypatch, "https://bob:pw@host/a") + assert code == 4 + assert "pw" not in stderr + assert "bob" not in stderr + assert "https://host/a" in stderr + + +def test_auth_error_carries_url_and_reason() -> None: + """``AuthRequiredError`` is an ``IngestError`` with exit 4.""" + error = AuthRequiredError("https://host/a", "401 Unauthorized") + assert isinstance(error, IngestError) + assert error.exit_code == 4 + assert error.url == "https://host/a" + assert error.reason == "401 Unauthorized" + assert str(error) == "Authentication required: https://host/a: 401 Unauthorized" + + +def test_login_url_patterns_settings(tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture) -> None: + """SPEC 5.1: default list, user override, and one bad regex skipped with a warning.""" + assert load_ingest_settings(tmp_path).login_url_patterns == list(DEFAULT_LOGIN_URL_PATTERNS) + root = _wiki(tmp_path, {"login_url_patterns": ["/auth/", "(", 7]}) + with caplog.at_level("WARNING"): + loaded = load_ingest_settings(root) + assert loaded.login_url_patterns == ["/auth/"] + assert sum("login_url_patterns" in record.getMessage() for record in caplog.records) == 2 + other = _wiki(tmp_path / "other", {"login_url_patterns": "not a list"}) + assert load_ingest_settings(other).login_url_patterns == list(DEFAULT_LOGIN_URL_PATTERNS) + + +def test_user_pattern_replaces_default(tmp_path: pathlib.Path) -> None: + """A user list replaces the default, so ``/login`` is no longer a hit.""" + root = _wiki(tmp_path, {"login_url_patterns": ["/gate/"]}) + pipeline = default_pipeline(root) + body = b"

Some ordinary page text for the test

" + pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://host/login")) + assert pipeline.run("https://host/a").exit_code == 0 + pipeline.fetchers[0] = UrlFetcher(pipeline.registry, opener=_Opener(body, final_url="https://HOST/Gate/x")) + with pytest.raises(AuthRequiredError, match="redirected to"): + pipeline.run("https://host/b") + + +def test_origin_sets_provenance_and_base_url(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """SPEC 7.1.5.2.9: a saved file keeps the URL it came from, with no network call.""" + root = _wiki(tmp_path) + saved = root / "ingest" / "console.html" + saved.write_text( + '

Build output that was saved by the user here

', + encoding="utf-8", + ) + opener = _Opener(status=500) + code, stdout, stderr = _cli( + root, + opener, + monkeypatch, + str(saved), + "--origin", + "https://bob:pw@host/job/x/console", + "--json", + ) + assert code == 0, stderr + assert opener.calls == 0 + payload = json.loads(stdout) + assert payload["origin"] == "https://host/job/x/console" + assert payload["origin_kind"] == "url" + page = root / payload["page_path"] + assert page.name == "console.md" + front = _front(page) + assert front["ingest"]["origin"] == "https://host/job/x/console" + assert front["ingest"]["origin_kind"] == "url" + assert front["ingest"]["final_url"] is None + assert "https://host/a" in page.read_text(encoding="utf-8") + log = (root / "wiki" / "log.md").read_text(encoding="utf-8") + assert "ingest https://host/job/x/console -> raw/articles/console.html" in log + + +def test_origin_human_output_has_no_fetched_line(tmp_path: pathlib.Path) -> None: + """A local file with ``--origin`` was not downloaded, so there is no ``Fetched:`` line.""" + root = _wiki(tmp_path) + saved = tmp_path / "notes.txt" + saved.write_text("saved text\n", encoding="utf-8") + result = default_pipeline(root).run(str(saved), origin="https://host/notes") + assert not any(line.startswith("Fetched:") for line in result.messages) + + +def test_origin_rejects_url_source_and_bad_scheme(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: + """SPEC 7.1.5.2.9.1: both checks exit 1 before any fetch.""" + root = _wiki(tmp_path) + before = _hashes(root) + opener = _Opener(b"x") + code, _stdout, stderr = _cli(root, opener, monkeypatch, "https://host/a", "--origin", "https://host/b") + assert code == 1 + assert "Error: --origin needs a local file: https://host/a" in stderr + saved = tmp_path / "a.txt" + saved.write_text("a\n", encoding="utf-8") + code, _stdout, stderr = _cli(root, None, monkeypatch, str(saved), "--origin", "ftp://host/a") + assert code == 1 + assert "Error: --origin must be an http or https URL: ftp://host/a" in stderr + code, _stdout, stderr = _cli(root, None, monkeypatch, str(saved), "--origin", "host/a") + assert code == 1 + assert "--origin must be an http or https URL" in stderr + assert opener.calls == 0 + assert _hashes(root) == before diff --git a/tests/test_ingest_edges.py b/tests/test_ingest_edges.py index 8a7d3ac..2f504db 100644 --- a/tests/test_ingest_edges.py +++ b/tests/test_ingest_edges.py @@ -1,4 +1,4 @@ -"""Branch coverage for wiki ingest stages. +"""Branch coverage for wiki ingest stages. Generated by AI/LLM: 2026-09-30 - error paths and converter edges from SPEC 7.1.5. """ @@ -41,7 +41,7 @@ from fiona.wiki.ingest.converters.pdf import ( _render, _walk_outline, ) -from fiona.wiki.ingest.fetchers import UrlFetcher, _without_userinfo, url_scheme +from fiona.wiki.ingest.fetchers import UrlFetcher, url_scheme, without_userinfo from fiona.wiki.ingest.log_writer import IngestLog from fiona.wiki.ingest.models import ArchiveResult, ConversionResult, FetchedSource, IngestError from fiona.wiki.ingest.pipeline import ( @@ -205,7 +205,11 @@ def test_markdown_frontmatter_edges(tmp_path: pathlib.Path) -> None: def test_html_edges(tmp_path: pathlib.Path) -> None: - """Comments, empty pages, og:title, images, and meta tags.""" + """Comments, empty pages, og:title, images, and meta tags. + + Modified by AI/LLM: 2026-10-07 - the block sweep drops a block with no text, + so the image sits in a paragraph, and a blank ``

`` no longer hides a later one. + """ converter = HtmlConverter() html = """ @@ -220,7 +224,7 @@ def test_html_edges(tmp_path: pathlib.Path) -> None:

-

Real

pic
+

Real

The picture pic shows the site.

""" path = tmp_path / "page.html" @@ -411,8 +415,8 @@ def test_download_and_log_edges(tmp_path: pathlib.Path, monkeypatch: pytest.Monk fetcher = UrlFetcher(registry, opener=_Opener(error="timed out")) with pytest.raises(IngestError, match="timed out"): fetcher.fetch("https://host/a.txt", root / "ingest") - assert _without_userinfo("https://user:secret@host:443/a.txt") == "https://host:443/a.txt" - assert _without_userinfo("https://host/a.txt") == "https://host/a.txt" + assert without_userinfo("https://user:secret@host:443/a.txt") == "https://host:443/a.txt" + assert without_userinfo("https://host/a.txt") == "https://host/a.txt" original_write = pathlib.Path.write_bytes diff --git a/tests/test_run_tests.py b/tests/test_run_tests.py index db73415..0fee14c 100644 --- a/tests/test_run_tests.py +++ b/tests/test_run_tests.py @@ -973,7 +973,7 @@ class TestRunTestsMain: run_tests.main() mock_runner.run_all_tests.assert_called_once_with( - False, True, True + True, False, True ) @patch('fiona.run_tests.TestRunner') diff --git a/tests/test_template_docs.py b/tests/test_template_docs.py index 4c587cc..2c9a845 100644 --- a/tests/test_template_docs.py +++ b/tests/test_template_docs.py @@ -17,6 +17,12 @@ TEMPLATE = llm_wiki_template() AGENTS = TEMPLATE / "AGENTS.md" SKILL = TEMPLATE / "utils" / "skills" / "fiona" / "SKILL.md" LOG = TEMPLATE / "wiki" / "log.md" +MEMORY = TEMPLATE / "MEMORY.md" +MEMORY_SKILL = TEMPLATE / "utils" / "skills" / "memory" / "SKILL.md" +MEMORIES = TEMPLATE / "utils" / "skills" / "memory" / "MEMORIES.md" +MEMORY_FILES = pytest.mark.parametrize("path", [MEMORY, MEMORIES], ids=["MEMORY", "MEMORIES"]) +HEADING = re.compile(r"^(#{2,6}) (\S+) ") +LIST_ITEM = re.compile(r"^\s*- (\[[ x]\] )?(\*\*([\d.]+)\*\*)?") def _body(path: pathlib.Path) -> str: @@ -66,3 +72,111 @@ def test_seed_log_has_one_entry() -> None: """ bullets = [line for line in _body(LOG).splitlines() if line.startswith("- ")] assert bullets == ["- DATE_PLACEHOLDER - Initialized the llm-wiki."] + + +def _memory_lines(path: pathlib.Path = MEMORY) -> list[str]: + """Return memory body lines outside the TOC block and code fences. + + Generated by AI/LLM: 2026-10-14 - sectionize ids skip TOC and code. + Modified by AI/LLM: 2026-10-15 - any memory file (SPEC 3.5.2). + """ + lines, skip = [], False + for line in _body(path).splitlines(): + if line.startswith(("` and ``, + each on its own line. Join blocks with one blank line + - [x] Report (SPEC 7.1.5.4.5.6.7): append + `html sweep: keep, review, drop` to the warnings. + No `keep` and at least one `review` -> `partial` with the + second warning. The empty-body rule of 7.1.5.4.5.5 still + runs first +- [x] Wire it into `html.py`. Run `_absolutize` on `` before + the sweep, so link rewriting covers every block. Pass `body` + instead of the old root to `_html_title`. Remove the + `article`/`main` root line +- [x] Check the page passes the wiki's own lint: the raw page goes + through `mdoctor fix` after writing. Confirm the markers stay + on their own lines, MD033 allows HTML comments, and no blank + line inside a marker pair is collapsed. Add one test that runs + `wiki sanity` on an ingested fixture with review blocks +- [x] Fixtures under `tests/fixtures/html/` (or inline strings, + matching `tests/test_ingest_edges.py`), one test per SPEC + acceptance of 7.1.5.4.5.6: + - [x] `landing.html`: no `
`, section with `

` and a + twelve-word `
`, then two `
` cards. All present, + in order (the agent-harness case) + - [x] `boilerplate.html`: nav list, footer paragraph, + `•`, `Share` twice, twelve-word paragraph + twice, all-link list, two-word `
`. Check the verdicts + and reasons `chrome`, `repeat`, `links`, `short` + - [x] `encyclopedia.html`: side `nav`, infobox, paragraphs with + `` citations, reference list, navbox table + - [x] `qa.html`: question, two answers with the same `pre`, vote + counts `42`, `7`, `3`, related `aside` + - [x] `forum.html`: custom-element posts, one in + `