feat: add Hermes Agent support (harness adapter, plugin, installer)
- harnesses/hermes.py: Hermes adapter reading state.db (session_model_usage cumulative counters) for authoritative session totals plus the plugin's JSONL ledger for per-turn breakdowns; Hermes's disjoint CanonicalUsage buckets are folded into the tool's inclusive (Vibe-style) contract - integration/hermes/plugin: Hermes plugin (plugin.yaml + register(ctx)) wiring pre_llm_call (turn labels), post_api_request + post_auxiliary_call (per-call ledger, queue + drainer, no hook-path I/O), post_llm_call (cost line after every interactive CLI turn; silent for subagents/cron/one-shot) - integration/hermes/install.py: standalone idempotent installer (validates the home, copies the plugin + pricing.json, runs hermes plugins enable); also wired as agent-cost-estimator --integrate hermes / --rm-integrate hermes - pricing: plugin-local pricing.json first (stdlib-only reader), then the package table, then default rates - tests: 22 new (52 total) covering adapter, plugin, installer - docs: SPEC.md 1.1.0 (sections 5.2.1, 7.3), SKILL.md, wiki entity Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -42,6 +42,17 @@ And add the cost information to this file for the current session.
|
|||||||
|
|
||||||
### Main Branch
|
### Main Branch
|
||||||
|
|
||||||
|
#### (pending) - feat: add Hermes Agent support (harness adapter, plugin, installer)
|
||||||
|
- **Date**: 2026-10-24
|
||||||
|
- **Model**: zai:zai-glm-5-3 (Hermes Agent)
|
||||||
|
- **Session Cost**: $11.49
|
||||||
|
- **Turn Cost**: $0.73 (running total at commit time)
|
||||||
|
- **Total for commit**: $11.49
|
||||||
|
- **Notes**: Hermes integration: harnesses/hermes.py adapter (state.db + plugin ledger),
|
||||||
|
integration/hermes/plugin (4 lifecycle hooks, per-turn cost line), standalone installer,
|
||||||
|
22 new tests (52 total), SPEC/SKILL/wiki docs updated. Cost via
|
||||||
|
`agent-cost-estimator --harness hermes` (this very session).
|
||||||
|
|
||||||
#### fb2ceae - Bump version to 0.2.0
|
#### fb2ceae - Bump version to 0.2.0
|
||||||
- **Date**: 2026-10-24
|
- **Date**: 2026-10-24
|
||||||
- **Model**: mistral:mistral-medium-3.5
|
- **Model**: mistral:mistral-medium-3.5
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
include README.md
|
include README.md
|
||||||
include LICENSE
|
include LICENSE
|
||||||
|
include pricing.json
|
||||||
recursive-include agent_cost_estimator/skill *.md
|
recursive-include agent_cost_estimator/skill *.md
|
||||||
|
recursive-include agent_cost_estimator/integration *.py *.yaml *.json
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
name: Agent Cost Estimator Specification
|
name: Agent Cost Estimator Specification
|
||||||
version: 1.0.0
|
version: 1.1.0
|
||||||
title: Project Spec Sheet
|
title: Project Spec Sheet
|
||||||
type: specification
|
type: specification
|
||||||
created: 2026-10-24
|
created: 2026-10-24
|
||||||
@@ -204,7 +204,8 @@ agent-cost-estimator/
|
|||||||
│ ├── harnesses/
|
│ ├── harnesses/
|
||||||
│ │ ├── __init__.py # Harness registry
|
│ │ ├── __init__.py # Harness registry
|
||||||
│ │ ├── base.py # HarnessAdapter base class (ABC)
|
│ │ ├── base.py # HarnessAdapter base class (ABC)
|
||||||
│ │ └── vibe.py # Vibe-specific implementation (CLI + VSCodium)
|
│ │ ├── vibe.py # Vibe-specific implementation (CLI + VSCodium)
|
||||||
|
│ │ └── hermes.py # Hermes Agent (state.db + plugin ledger)
|
||||||
│ │
|
│ │
|
||||||
│ ├── hooks/
|
│ ├── hooks/
|
||||||
│ │ ├── mistral-vibe-cli/
|
│ │ ├── mistral-vibe-cli/
|
||||||
@@ -213,6 +214,13 @@ agent-cost-estimator/
|
|||||||
│ │ │ └── post_turn.py # Hook for Mistral Vibe VSCodium extension
|
│ │ │ └── post_turn.py # Hook for Mistral Vibe VSCodium extension
|
||||||
│ │ └── README.md # Overview of hook structure
|
│ │ └── README.md # Overview of hook structure
|
||||||
│ │
|
│ │
|
||||||
|
│ ├── integration/
|
||||||
|
│ │ └── hermes/
|
||||||
|
│ │ ├── install.py # Standalone installer (any Hermes home)
|
||||||
|
│ │ └── plugin/ # THE Hermes plugin (copied by install.py)
|
||||||
|
│ │ ├── plugin.yaml
|
||||||
|
│ │ └── __init__.py
|
||||||
|
│ │
|
||||||
│ └── skill/
|
│ └── skill/
|
||||||
│ └── SKILL.md # LLM usage instructions
|
│ └── SKILL.md # LLM usage instructions
|
||||||
│
|
│
|
||||||
@@ -504,6 +512,7 @@ agent-cost-estimator = "agent_cost_estimator.cli:main"
|
|||||||
```toml
|
```toml
|
||||||
[project.entry-points."agent_cost_estimator.harnesses"]
|
[project.entry-points."agent_cost_estimator.harnesses"]
|
||||||
vibe = "agent_cost_estimator.harnesses.vibe:VibeHarness"
|
vibe = "agent_cost_estimator.harnesses.vibe:VibeHarness"
|
||||||
|
hermes = "agent_cost_estimator.harnesses.hermes:HermesHarness"
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4.3. Installation
|
### 4.3. Installation
|
||||||
@@ -516,7 +525,7 @@ pip install .
|
|||||||
|
|
||||||
**Verify:**
|
**Verify:**
|
||||||
```bash
|
```bash
|
||||||
agent-cost-estimator --list-harnesses # Should output: vibe
|
agent-cost-estimator --list-harnesses # Should output: hermes, vibe
|
||||||
agent-cost-estimator --skill # Should output SKILL.md
|
agent-cost-estimator --skill # Should output SKILL.md
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -532,7 +541,7 @@ agent-cost-estimator --skill # Should output SKILL.md
|
|||||||
|
|
||||||
| Option | Type | Required | Default | Description |
|
| Option | Type | Required | Default | Description |
|
||||||
|--------|------|----------|---------|-------------|
|
|--------|------|----------|---------|-------------|
|
||||||
| `--harness` | str | No | `vibe` | Harness to read (vibe, etc.) |
|
| `--harness` | str | No | `vibe` | Harness to read (vibe, hermes) |
|
||||||
| `--session-id` | str | No | auto | Session ID or directory path |
|
| `--session-id` | str | No | auto | Session ID or directory path |
|
||||||
| `--home` | str | No | harness default | Harness data root directory |
|
| `--home` | str | No | harness default | Harness data root directory |
|
||||||
| `--pricing` | str | No | bundled | Path to pricing.json file |
|
| `--pricing` | str | No | bundled | Path to pricing.json file |
|
||||||
@@ -586,6 +595,76 @@ agent-cost-estimator --skill # Should output SKILL.md
|
|||||||
- Cached tokens are clamped to input tokens (matching Vibe's behavior)
|
- Cached tokens are clamped to input tokens (matching Vibe's behavior)
|
||||||
- Fresh tokens = input_tokens - cached_input_tokens
|
- Fresh tokens = input_tokens - cached_input_tokens
|
||||||
|
|
||||||
|
### 5.2.1. Hermes Harness Adapter
|
||||||
|
|
||||||
|
**Status:** Implemented
|
||||||
|
|
||||||
|
**Purpose:** Read session data from Hermes Agent's SQLite store.
|
||||||
|
|
||||||
|
**File:** `agent_cost_estimator/harnesses/hermes.py`
|
||||||
|
|
||||||
|
**Implementation:**
|
||||||
|
|
||||||
|
- `default_home()`: Returns `$HERMES_HOME` or `~/.hermes`
|
||||||
|
- `resolve_session_dir()`: Accepts a session id
|
||||||
|
(e.g. `local-<hash>`), a `state.db` path, or a profile home
|
||||||
|
directory; None picks the session with the newest
|
||||||
|
`last_activity_at`. Returns a `HermesSession`
|
||||||
|
(db_path + session_id; `.name` exposes the session id for the
|
||||||
|
CLI's display).
|
||||||
|
- `read_meta()`: Reads `sessions.title` and `sessions.model`
|
||||||
|
- `collect_usage()`: session totals from
|
||||||
|
`session_model_usage` (authoritative cumulative counter);
|
||||||
|
per-turn breakdown from the plugin's usage ledger
|
||||||
|
|
||||||
|
**Session Discovery:**
|
||||||
|
|
||||||
|
1. If `--session-id` is a session id: validate against `sessions`
|
||||||
|
2. If it is a file: treat as `state.db`
|
||||||
|
3. If it is a directory: `<dir>/state.db`
|
||||||
|
4. Otherwise: newest `last_activity_at` in `sessions`
|
||||||
|
|
||||||
|
**Usage Source Priority:**
|
||||||
|
|
||||||
|
1. `state.db` `session_model_usage` (cumulative, written by
|
||||||
|
Hermes after every API call - survives ledger loss)
|
||||||
|
2. The plugin's JSONL ledger
|
||||||
|
(`<home>/plugin-data/agent-cost-estimator/usage/<session_id>.jsonl`,
|
||||||
|
per-call records grouped by `turn_id`) - only present once the
|
||||||
|
plugin is installed
|
||||||
|
|
||||||
|
**Token Bucket Mapping (Hermes -> tool contract):**
|
||||||
|
|
||||||
|
Hermes's `CanonicalUsage` buckets are disjoint; the tool's
|
||||||
|
`SessionUsage` is Vibe-style (input includes cached):
|
||||||
|
|
||||||
|
| Hermes column | SessionUsage |
|
||||||
|
|----------------------|--------------------------|
|
||||||
|
| `input_tokens` | fresh input (excl. cache)|
|
||||||
|
| `cache_read_tokens` | cached input |
|
||||||
|
| `cache_write_tokens` | fresh input (billed) |
|
||||||
|
|
||||||
|
- `input_tokens` = input + cache_read + cache_write
|
||||||
|
(= `prompt_tokens`)
|
||||||
|
- `cached_input_tokens` = cache_read
|
||||||
|
- `fresh_input_tokens` = input + cache_write
|
||||||
|
- `output_tokens` = output (includes reasoning tokens)
|
||||||
|
|
||||||
|
**Ledger Format (one JSON object per line):**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"ts": 1792847431.9, "session_id": "local-...", "turn_id": "t1",
|
||||||
|
"model": "zai-glm-5-3", "provider": "zai",
|
||||||
|
"api_request_id": "r1", "aux_task": null,
|
||||||
|
"label": "user message head", "calls": 1,
|
||||||
|
"input_tokens": 453000, "cached_input_tokens": 400000,
|
||||||
|
"output_tokens": 9000}
|
||||||
|
```
|
||||||
|
|
||||||
|
Records are deduplicated by `api_request_id` when present;
|
||||||
|
`cached_input_tokens` is clamped to `input_tokens` per record and
|
||||||
|
per turn.
|
||||||
|
|
||||||
### 5.3. Pricing Configuration
|
### 5.3. Pricing Configuration
|
||||||
|
|
||||||
**Status:** Implemented
|
**Status:** Implemented
|
||||||
@@ -681,24 +760,46 @@ def compute_cost(tokens: dict[str, int], rates: dict[str, float]) -> dict[str, f
|
|||||||
**Command:** `agent-cost-estimator --integrate <harness>`
|
**Command:** `agent-cost-estimator --integrate <harness>`
|
||||||
|
|
||||||
**Supported Harnesses:**
|
**Supported Harnesses:**
|
||||||
- `mistral` (Vibe CLI)
|
- `mistral`, `mistral-vibe-cli`, `vibe`, `cli` (Vibe CLI)
|
||||||
|
- `mistral-vibe-code`, `vscodium`, `code` (VSCodium extension)
|
||||||
|
- `hermes`, `hermes-agent` (Hermes Agent plugin)
|
||||||
|
|
||||||
**Modular Design:**
|
**Modular Design:**
|
||||||
|
|
||||||
The `--integrate` command accepts a harness name argument, allowing future
|
The `--integrate` command accepts a harness name argument; each
|
||||||
support for additional harnesses without breaking changes.
|
harness dispatches to its own installer (Vibe family: inline hook
|
||||||
|
install; Hermes: `agent_cost_estimator.integration.hermes.install`).
|
||||||
|
|
||||||
**Actions:**
|
**Actions (Vibe family):**
|
||||||
|
|
||||||
1. Creates harness-specific hook directory
|
1. Creates harness-specific hook directory
|
||||||
2. Installs hook script
|
2. Installs hook script
|
||||||
3. Creates hooks.toml configuration
|
3. Creates hooks.toml configuration
|
||||||
4. Updates bash allowlist in config.toml
|
4. Updates bash allowlist in config.toml
|
||||||
|
|
||||||
**Example:**
|
**Actions (Hermes):**
|
||||||
|
|
||||||
|
1. Validates the Hermes home (`state.db` or `config.yaml` present)
|
||||||
|
2. Copies the plugin (`plugin.yaml`, `__init__.py`) to
|
||||||
|
`<home>/plugins/agent-cost-estimator/`
|
||||||
|
3. Ships `pricing.json` next to the plugin (preserves an
|
||||||
|
already-tuned installed table on reinstall)
|
||||||
|
4. Runs `hermes plugins enable agent-cost-estimator` (prints the
|
||||||
|
manual step when the `hermes` CLI is not on PATH)
|
||||||
|
|
||||||
|
**Examples:**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
agent-cost-estimator --integrate mistral
|
agent-cost-estimator --integrate mistral
|
||||||
|
agent-cost-estimator --integrate hermes
|
||||||
|
agent-cost-estimator --rm-integrate hermes
|
||||||
|
```
|
||||||
|
|
||||||
|
The Hermes installer is also runnable standalone on any machine:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m agent_cost_estimator.integration.hermes.install
|
||||||
|
[--hermes-home DIR] [--uninstall] [--dry-run]
|
||||||
```
|
```
|
||||||
|
|
||||||
**Future Extension:**
|
**Future Extension:**
|
||||||
@@ -949,6 +1050,133 @@ engine but may have differences in:
|
|||||||
|
|
||||||
The hook is designed to be resilient to these differences.
|
The hook is designed to be resilient to these differences.
|
||||||
|
|
||||||
|
### 7.3. Hermes Agent (Nous Research) Configuration
|
||||||
|
|
||||||
|
**Harness:** Hermes Agent
|
||||||
|
**Adapter:** `agent_cost_estimator.harnesses.hermes:HermesHarness`
|
||||||
|
**Plugin Location:** `agent_cost_estimator/integration/hermes/plugin/`
|
||||||
|
**Installer:** `agent_cost_estimator/integration/hermes/install.py`
|
||||||
|
|
||||||
|
Hermes integration is a **plugin**, not a hook script: Hermes
|
||||||
|
loads plugins from `<home>/plugins/<name>/` (manifest
|
||||||
|
`plugin.yaml` + `__init__.py` with `register(ctx)`), gated by
|
||||||
|
`hermes plugins enable <name>`.
|
||||||
|
|
||||||
|
#### 7.3.1. Plugin Implementation
|
||||||
|
|
||||||
|
**File:** `agent_cost_estimator/integration/hermes/plugin/__init__.py`
|
||||||
|
|
||||||
|
**Manifest:**
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
name: agent-cost-estimator
|
||||||
|
version: <package version>
|
||||||
|
provides_hooks:
|
||||||
|
- pre_llm_call
|
||||||
|
- post_api_request
|
||||||
|
- post_auxiliary_call
|
||||||
|
- post_llm_call
|
||||||
|
```
|
||||||
|
|
||||||
|
**Registered hooks and payloads:**
|
||||||
|
|
||||||
|
| Hook | Payload fields used | Action |
|
||||||
|
|---|---|---|
|
||||||
|
| `pre_llm_call` | `turn_id`, `user_message` | capture the turn label (message head, 60 chars) |
|
||||||
|
| `post_api_request` | `session_id`, `turn_id`, `model`, `provider`, `api_request_id`, `usage` (CanonicalUsage dict) | queue one ledger record |
|
||||||
|
| `post_auxiliary_call` | same + `aux_task` | queue one ledger record (aux spend folds into the parent turn) |
|
||||||
|
| `post_llm_call` | `session_id`, `turn_id`, `model`, `platform` | print the cost line |
|
||||||
|
|
||||||
|
**Data flow:**
|
||||||
|
|
||||||
|
1. Each API call appends a JSON line (via a bounded in-process
|
||||||
|
queue and a single daemon drainer thread - the hook callback
|
||||||
|
itself never does I/O) to
|
||||||
|
`<home>/plugin-data/agent-cost-estimator/usage/<session_id>.jsonl`
|
||||||
|
2. The per-turn cost also accumulates in memory, so the printed
|
||||||
|
turn figure is exact even before the ledger flushes
|
||||||
|
3. At turn end (`post_llm_call` - which Hermes fires after
|
||||||
|
flushing its token queue to state.db), the plugin reads the
|
||||||
|
session's cumulative totals from `state.db`
|
||||||
|
(`session_model_usage`, read-only) and prints:
|
||||||
|
|
||||||
|
```
|
||||||
|
-- provider:model | session:$X.XX | turn:$Y.YY
|
||||||
|
```
|
||||||
|
|
||||||
|
4. Session totals come from state.db (authoritative, like Vibe's
|
||||||
|
projection-state); the ledger provides the per-turn breakdown
|
||||||
|
across restarts (the tool's `--harness hermes` adapter reads
|
||||||
|
both).
|
||||||
|
|
||||||
|
**Print gating:**
|
||||||
|
|
||||||
|
- Prints only for `platform == "cli"` (the interactive terminal)
|
||||||
|
- Never prints for subagents (`platform == "subagent"`), cron
|
||||||
|
runs, or one-shot runs (`HERMES_SINGLE_QUERY_SESSION=1`,
|
||||||
|
i.e. `hermes chat -q`) - their stdout must stay
|
||||||
|
machine-readable
|
||||||
|
- Subagent and auxiliary calls still RECORD to the ledger (their
|
||||||
|
spend is real and shows in session totals)
|
||||||
|
|
||||||
|
**Pricing resolution (per print):**
|
||||||
|
|
||||||
|
1. `pricing.json` next to the installed plugin (stdlib-only
|
||||||
|
reader; tune rates by editing this file)
|
||||||
|
2. the `agent_cost_estimator` package's bundled `pricing.json`
|
||||||
|
(when importable in Hermes's interpreter)
|
||||||
|
3. hard-coded defaults (1.5 / 0.15 / 7.5 USD per M) with the
|
||||||
|
`model price not available` warning
|
||||||
|
|
||||||
|
**Failure policy:** every callback swallows its own errors
|
||||||
|
(debug-logged); a cost line must never break a turn. Ledger
|
||||||
|
writes are best-effort (bounded queue, drop on overflow - session
|
||||||
|
totals are unaffected).
|
||||||
|
|
||||||
|
**Known limitation:** Hermes skips a timeout-bounded hook
|
||||||
|
callback that is still running for the same `turn_id`, so under
|
||||||
|
MoA advisor fan-out a concurrently-fired duplicate call may not be
|
||||||
|
ledgered. Session totals are unaffected; only that call's
|
||||||
|
contribution to the per-turn breakdown can be lost.
|
||||||
|
|
||||||
|
#### 7.3.2. Installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# automatic (from the package)
|
||||||
|
agent-cost-estimator --integrate hermes
|
||||||
|
|
||||||
|
# standalone (any machine, source checkout, or installed package)
|
||||||
|
python -m agent_cost_estimator.integration.hermes.install
|
||||||
|
python path/to/install.py --hermes-home /custom/home --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
The installer:
|
||||||
|
|
||||||
|
1. Validates the target home (`state.db` or `config.yaml`)
|
||||||
|
2. Copies `plugin.yaml` + `__init__.py` to
|
||||||
|
`<home>/plugins/agent-cost-estimator/`
|
||||||
|
3. Copies the bundled `pricing.json` next to the plugin,
|
||||||
|
preserving an already-installed (user-tuned) table on
|
||||||
|
reinstall
|
||||||
|
4. Runs `hermes plugins enable agent-cost-estimator`; on failure
|
||||||
|
or missing CLI prints the manual command
|
||||||
|
|
||||||
|
Removal:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
agent-cost-estimator --rm-integrate hermes
|
||||||
|
python -m agent_cost_estimator.integration.hermes.install --uninstall
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 7.3.3. Verification
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# after install + a Hermes restart + one turn:
|
||||||
|
agent-cost-estimator --harness hermes # session + latest turn
|
||||||
|
agent-cost-estimator --harness hermes -d # full table
|
||||||
|
hermes plugins list --plain --no-bundled # shows the enabled plugin
|
||||||
|
```
|
||||||
|
|
||||||
## 8. Error Handling
|
## 8. Error Handling
|
||||||
|
|
||||||
**Principle:** Fail gracefully, never break the user's session.
|
**Principle:** Fail gracefully, never break the user's session.
|
||||||
@@ -979,13 +1207,19 @@ The hook is designed to be resilient to these differences.
|
|||||||
**Coverage:** All core functionality tested
|
**Coverage:** All core functionality tested
|
||||||
|
|
||||||
**Test Files:**
|
**Test Files:**
|
||||||
- `tests/test_agent_cost_estimator.py` (30 tests)
|
- `tests/test_agent_cost_estimator.py` (52 tests)
|
||||||
|
|
||||||
**Test Categories:**
|
**Test Categories:**
|
||||||
|
|
||||||
- Core cost calculation
|
- Core cost calculation
|
||||||
- Pricing table loading/lookup/validation
|
- Pricing table loading/lookup/validation
|
||||||
- Vibe harness adapter (journal parsing, projection, dedupe, session discovery)
|
- Vibe harness adapter (journal parsing, projection, dedupe, session discovery)
|
||||||
|
- Hermes harness adapter (state.db totals, ledger fallback, dedupe,
|
||||||
|
clamp, session discovery, CLI --json end to end)
|
||||||
|
- Hermes plugin (hook registration, CanonicalUsage bucket folding,
|
||||||
|
record + print, subagent/single-query gating, pricing sources)
|
||||||
|
- Hermes installer (dry-run, home validation, plugin + pricing copy,
|
||||||
|
uninstall)
|
||||||
- Harness registry
|
- Harness registry
|
||||||
- CLI (pricing flags, -t, -s, --json, --update-pricing, --integrate, error exits)
|
- CLI (pricing flags, -t, -s, --json, --update-pricing, --integrate, error exits)
|
||||||
|
|
||||||
@@ -1106,12 +1340,23 @@ agent-cost-estimator --skill
|
|||||||
| `harnesses/__init__.py` | Registry | Implemented |
|
| `harnesses/__init__.py` | Registry | Implemented |
|
||||||
| `harnesses/base.py` | Base interface | Implemented |
|
| `harnesses/base.py` | Base interface | Implemented |
|
||||||
| `harnesses/vibe.py` | Mistral Vibe CLI | Implemented |
|
| `harnesses/vibe.py` | Mistral Vibe CLI | Implemented |
|
||||||
|
| `harnesses/hermes.py` | Hermes Agent | Implemented |
|
||||||
|
|
||||||
**Hooks:**
|
**Hooks:**
|
||||||
|
|
||||||
| File | Purpose |
|
| File | Purpose | Status |
|
||||||
|------|---------|
|
|------|---------|--------|
|
||||||
| `hooks/post_turn.py` | Vibe post-agent hook | Implemented |
|
| `hooks/post_turn.py` | Vibe post-agent hook | Implemented |
|
||||||
|
| `hooks/mistral-vibe-cli/post_turn.py` | Vibe CLI hook | Implemented |
|
||||||
|
| `hooks/mistral-vibe-code/post_turn.py` | VSCodium hook | Implemented |
|
||||||
|
|
||||||
|
**Integrations:**
|
||||||
|
|
||||||
|
| File | Purpose | Status |
|
||||||
|
|------|---------|--------|
|
||||||
|
| `integration/hermes/plugin/plugin.yaml` | Hermes plugin manifest | Implemented |
|
||||||
|
| `integration/hermes/plugin/__init__.py` | Hermes plugin (hooks + cost line) | Implemented |
|
||||||
|
| `integration/hermes/install.py` | Standalone Hermes installer | Implemented |
|
||||||
|
|
||||||
**Configuration:**
|
**Configuration:**
|
||||||
|
|
||||||
|
|||||||
+29
-24
@@ -46,8 +46,9 @@ def parse_args(argv):
|
|||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--integrate", type=str, default=None, metavar="HARNESS",
|
"--integrate", type=str, default=None, metavar="HARNESS",
|
||||||
help="install hooks for the specified harness "
|
help="install hooks/plugins for the specified harness "
|
||||||
"(e.g., 'mistral', 'mistral-vibe-cli', 'mistral-vibe-code')",
|
"(e.g., 'mistral', 'mistral-vibe-cli', "
|
||||||
|
"'mistral-vibe-code', 'hermes')",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--session-id", default=None,
|
"--session-id", default=None,
|
||||||
@@ -108,33 +109,27 @@ def parse_args(argv):
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--rm-integrate", type=str, default=None, metavar="HARNESS",
|
"--rm-integrate", type=str, default=None, metavar="HARNESS",
|
||||||
help="remove hooks integration for the specified harness "
|
help="remove hooks integration for the specified harness "
|
||||||
"(e.g., 'mistral-vibe-cli', 'mistral-vibe-code')",
|
"(e.g., 'mistral-vibe-cli', 'mistral-vibe-code', 'hermes')",
|
||||||
)
|
)
|
||||||
return parser.parse_args(argv)
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
def integrate_hooks(harness_name):
|
def integrate_hooks(harness_name):
|
||||||
"""Install hooks for the specified harness.
|
"""Install hooks/plugins for the specified harness.
|
||||||
|
|
||||||
Currently supports:
|
Supported:
|
||||||
- mistral-vibe-cli: Installs hooks for Mistral Vibe CLI
|
- mistral-vibe-cli: Mistral Vibe CLI hooks
|
||||||
- mistral-vibe-code: Installs hooks for Mistral Vibe VSCodium extension
|
- mistral-vibe-code: Mistral Vibe VSCodium extension hooks
|
||||||
- mistral: Alias for mistral-vibe-cli (backward compatibility)
|
- hermes / hermes-agent: Hermes Agent plugin (via
|
||||||
|
agent_cost_estimator.integration.hermes.install)
|
||||||
This function:
|
- mistral, vibe, cli: aliases for mistral-vibe-cli
|
||||||
1. Creates harness-specific directories
|
- vscodium, code: aliases for mistral-vibe-code
|
||||||
2. Installs hook scripts from modular hook directory
|
|
||||||
3. Creates/updates hook configuration
|
|
||||||
4. Adds agent-cost-estimator to bash allowlist
|
|
||||||
|
|
||||||
Args:
|
|
||||||
harness_name: Name of the harness to integrate
|
|
||||||
|
|
||||||
Returns 0 on success, 1 on error.
|
Returns 0 on success, 1 on error.
|
||||||
"""
|
"""
|
||||||
import os
|
import os
|
||||||
import stat
|
import stat
|
||||||
|
|
||||||
# Validate harness name
|
# Validate harness name
|
||||||
# Map aliases to canonical names
|
# Map aliases to canonical names
|
||||||
harness_aliases = {
|
harness_aliases = {
|
||||||
@@ -143,9 +138,14 @@ def integrate_hooks(harness_name):
|
|||||||
"cli": "mistral-vibe-cli",
|
"cli": "mistral-vibe-cli",
|
||||||
"vscodium": "mistral-vibe-code",
|
"vscodium": "mistral-vibe-code",
|
||||||
"code": "mistral-vibe-code",
|
"code": "mistral-vibe-code",
|
||||||
|
"hermes-agent": "hermes",
|
||||||
}
|
}
|
||||||
canonical_name = harness_aliases.get(harness_name, harness_name)
|
canonical_name = harness_aliases.get(harness_name, harness_name)
|
||||||
|
|
||||||
|
if canonical_name == "hermes":
|
||||||
|
from .integration.hermes.install import main as install_hermes
|
||||||
|
return install_hermes()
|
||||||
|
|
||||||
supported_harnesses = ["mistral-vibe-cli", "mistral-vibe-code"]
|
supported_harnesses = ["mistral-vibe-cli", "mistral-vibe-code"]
|
||||||
if canonical_name not in supported_harnesses:
|
if canonical_name not in supported_harnesses:
|
||||||
print(f"error: unsupported harness '{harness_name}'. "
|
print(f"error: unsupported harness '{harness_name}'. "
|
||||||
@@ -317,10 +317,10 @@ allowlist = ["agent-cost-estimator", "python"]
|
|||||||
|
|
||||||
def remove_integrate_hooks(harness_name):
|
def remove_integrate_hooks(harness_name):
|
||||||
"""Remove hooks integration for the specified harness.
|
"""Remove hooks integration for the specified harness.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
harness_name: Name of the harness to remove
|
harness_name: Name of the harness to remove
|
||||||
|
|
||||||
Returns 0 on success, 1 on error.
|
Returns 0 on success, 1 on error.
|
||||||
"""
|
"""
|
||||||
# Map aliases to canonical names
|
# Map aliases to canonical names
|
||||||
@@ -330,9 +330,14 @@ def remove_integrate_hooks(harness_name):
|
|||||||
"cli": "mistral-vibe-cli",
|
"cli": "mistral-vibe-cli",
|
||||||
"vscodium": "mistral-vibe-code",
|
"vscodium": "mistral-vibe-code",
|
||||||
"code": "mistral-vibe-code",
|
"code": "mistral-vibe-code",
|
||||||
|
"hermes-agent": "hermes",
|
||||||
}
|
}
|
||||||
canonical_name = harness_aliases.get(harness_name, harness_name)
|
canonical_name = harness_aliases.get(harness_name, harness_name)
|
||||||
|
|
||||||
|
if canonical_name == "hermes":
|
||||||
|
from .integration.hermes.install import main as uninstall_hermes
|
||||||
|
return uninstall_hermes(["--uninstall"])
|
||||||
|
|
||||||
supported_harnesses = ["mistral-vibe-cli", "mistral-vibe-code"]
|
supported_harnesses = ["mistral-vibe-cli", "mistral-vibe-code"]
|
||||||
if canonical_name not in supported_harnesses:
|
if canonical_name not in supported_harnesses:
|
||||||
print(f"error: unsupported harness '{harness_name}'. "
|
print(f"error: unsupported harness '{harness_name}'. "
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import importlib.metadata
|
|||||||
import logging
|
import logging
|
||||||
|
|
||||||
from .base import HarnessAdapter, SessionMeta, SessionUsage, TurnUsage
|
from .base import HarnessAdapter, SessionMeta, SessionUsage, TurnUsage
|
||||||
|
from .hermes import HermesHarness
|
||||||
from .vibe import VibeHarness
|
from .vibe import VibeHarness
|
||||||
|
|
||||||
LOG = logging.getLogger("agent_cost_estimator.harnesses")
|
LOG = logging.getLogger("agent_cost_estimator.harnesses")
|
||||||
@@ -22,6 +23,7 @@ LOG = logging.getLogger("agent_cost_estimator.harnesses")
|
|||||||
ENTRY_POINT_GROUP = "agent_cost_estimator.harnesses"
|
ENTRY_POINT_GROUP = "agent_cost_estimator.harnesses"
|
||||||
|
|
||||||
BUILTIN_HARNESSES: dict[str, type[HarnessAdapter]] = {
|
BUILTIN_HARNESSES: dict[str, type[HarnessAdapter]] = {
|
||||||
|
HermesHarness.name: HermesHarness,
|
||||||
VibeHarness.name: VibeHarness,
|
VibeHarness.name: VibeHarness,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,6 +75,7 @@ __all__ = [
|
|||||||
"BUILTIN_HARNESSES",
|
"BUILTIN_HARNESSES",
|
||||||
"ENTRY_POINT_GROUP",
|
"ENTRY_POINT_GROUP",
|
||||||
"HarnessAdapter",
|
"HarnessAdapter",
|
||||||
|
"HermesHarness",
|
||||||
"SessionMeta",
|
"SessionMeta",
|
||||||
"SessionUsage",
|
"SessionUsage",
|
||||||
"TurnUsage",
|
"TurnUsage",
|
||||||
|
|||||||
@@ -0,0 +1,312 @@
|
|||||||
|
"""Hermes Agent (Nous Research) adapter.
|
||||||
|
|
||||||
|
Hermes stores session history in a SQLite database at
|
||||||
|
<home>/state.db (the profile home, ~/.hermes by default):
|
||||||
|
|
||||||
|
- sessions: one row per session (title, model, activity times)
|
||||||
|
- session_model_usage: cumulative per-model token counters,
|
||||||
|
updated after every API call - the authoritative session
|
||||||
|
totals, like Vibe's projection-state.json
|
||||||
|
|
||||||
|
Hermes does not persist a per-call or per-turn usage breakdown,
|
||||||
|
so per-turn numbers come from the agent-cost-estimator plugin's
|
||||||
|
JSONL ledger (<home>/plugin-data/agent-cost-estimator/usage/
|
||||||
|
<session_id>.jsonl), written by its post_api_request hook. When
|
||||||
|
the ledger is absent (plugin not installed / not loaded), only
|
||||||
|
session totals are reported and the latest turn reads as zero.
|
||||||
|
|
||||||
|
Token buckets (Hermes CanonicalUsage - DISJOINT buckets):
|
||||||
|
- input_tokens: fresh prompt tokens (EXCLUDES cache)
|
||||||
|
- cache_read_tokens: prompt tokens served from the read cache
|
||||||
|
- cache_write_tokens: prompt tokens written to the cache this
|
||||||
|
call (billed as fresh input at most providers)
|
||||||
|
SessionUsage mapping (Vibe-compatible: input INCLUDES cached):
|
||||||
|
- input_tokens: input + cache_read + cache_write
|
||||||
|
(= CanonicalUsage.prompt_tokens)
|
||||||
|
- cached_input_tokens: cache_read
|
||||||
|
- fresh_input_tokens: input + cache_write (billed at full rate)
|
||||||
|
- output_tokens: output (includes reasoning tokens)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import sqlite3
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from .base import HarnessAdapter, SessionMeta, SessionUsage, TurnUsage
|
||||||
|
|
||||||
|
LOG = logging.getLogger("agent_cost_estimator.harnesses.hermes")
|
||||||
|
|
||||||
|
DB_NAME = "state.db"
|
||||||
|
LEDGER_DIR = ("plugin-data", "agent-cost-estimator", "usage")
|
||||||
|
LEDGER_MAX_BYTES = 50 * 1024 * 1024 # hard cap per session ledger
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class HermesSession:
|
||||||
|
"""A resolved Hermes session: (state.db path, session id).
|
||||||
|
|
||||||
|
Named `name` so the CLI's `session_dir.name` keeps working;
|
||||||
|
`db_path` and `session_id` are the real handles. `session_id`
|
||||||
|
is None until the harness resolves it (path-style arguments
|
||||||
|
carry no id; None means "most recently active").
|
||||||
|
"""
|
||||||
|
|
||||||
|
db_path: pathlib.Path
|
||||||
|
session_id: str | None = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def name(self) -> str:
|
||||||
|
return self.session_id or self.db_path.parent.name
|
||||||
|
|
||||||
|
|
||||||
|
def default_home() -> pathlib.Path:
|
||||||
|
"""The Hermes profile home: $HERMES_HOME or ~/.hermes."""
|
||||||
|
env = os.environ.get("HERMES_HOME")
|
||||||
|
if env:
|
||||||
|
return pathlib.Path(env).expanduser()
|
||||||
|
return pathlib.Path.home() / ".hermes"
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_session_dir(session_arg, home):
|
||||||
|
"""Resolve a session argument to a HermesSession.
|
||||||
|
|
||||||
|
Accepts a session id (e.g. "local-<hash>"), a state.db path,
|
||||||
|
or a profile home directory. None picks the most recently
|
||||||
|
active session. Raises FileNotFoundError when nothing matches.
|
||||||
|
"""
|
||||||
|
home = pathlib.Path(home) if home else default_home()
|
||||||
|
db_path = home / DB_NAME
|
||||||
|
session_id = None
|
||||||
|
if session_arg:
|
||||||
|
candidate = pathlib.Path(session_arg).expanduser()
|
||||||
|
if candidate.is_file():
|
||||||
|
# A state.db (or sqlite file) path directly
|
||||||
|
db_path = candidate
|
||||||
|
elif candidate.is_dir():
|
||||||
|
if not (candidate / DB_NAME).is_file():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"no {DB_NAME} under {candidate}"
|
||||||
|
)
|
||||||
|
db_path = candidate / DB_NAME
|
||||||
|
else:
|
||||||
|
# Treat as a session id; validated below
|
||||||
|
session_id = str(session_arg)
|
||||||
|
return HermesSession(db_path=db_path, session_id=session_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _connect(db_path: pathlib.Path) -> sqlite3.Connection:
|
||||||
|
# Read-only URI: never take a write lock on the live DB.
|
||||||
|
return sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5.0)
|
||||||
|
|
||||||
|
|
||||||
|
def read_meta(session_dir, home=None):
|
||||||
|
"""Return (title, model_name, alias) for the session.
|
||||||
|
|
||||||
|
title comes from sessions.title; model_name is the session's
|
||||||
|
model (e.g. "zai-glm-5-3"); alias stays None (Hermes has no
|
||||||
|
separate alias concept).
|
||||||
|
"""
|
||||||
|
row = _fetch_one(
|
||||||
|
session_dir,
|
||||||
|
"SELECT title, model FROM sessions WHERE id = ?",
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
return None, None, None
|
||||||
|
title, model = row
|
||||||
|
return title, model, None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_ledger(session_dir, home=None):
|
||||||
|
"""Sum the plugin's per-call JSONL ledger for the session.
|
||||||
|
|
||||||
|
Each line is one API call:
|
||||||
|
{"turn_id": ..., "ts": ..., "model": ..., "provider": ...,
|
||||||
|
"input_tokens": N, "cached_input_tokens": N,
|
||||||
|
"output_tokens": N, "label": "..."}
|
||||||
|
|
||||||
|
Calls group into turn buckets keyed by turn_id; the bucket
|
||||||
|
label comes from the first call of the turn. Cached input is
|
||||||
|
clamped to input per call. Dedupes by api_request_id when
|
||||||
|
present (a retried call keeps one record per request).
|
||||||
|
"""
|
||||||
|
ledger_path = _ledger_path(session_dir)
|
||||||
|
totals = {"input": 0, "cached": 0, "output": 0, "calls": 0}
|
||||||
|
turns: dict[str, dict] = {}
|
||||||
|
order: list[str] = []
|
||||||
|
if not ledger_path.is_file():
|
||||||
|
return totals, []
|
||||||
|
seen = set()
|
||||||
|
with ledger_path.open(encoding="utf-8") as handle:
|
||||||
|
for line in handle:
|
||||||
|
try:
|
||||||
|
rec = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
LOG.warning("skipping malformed ledger line in %s",
|
||||||
|
ledger_path.name)
|
||||||
|
continue
|
||||||
|
req_id = rec.get("api_request_id")
|
||||||
|
if req_id:
|
||||||
|
if req_id in seen:
|
||||||
|
continue
|
||||||
|
seen.add(req_id)
|
||||||
|
turn_id = str(rec.get("turn_id") or "(no turn)")
|
||||||
|
inp = int(rec.get("input_tokens") or 0)
|
||||||
|
cached = min(int(rec.get("cached_input_tokens") or 0), inp)
|
||||||
|
out = int(rec.get("output_tokens") or 0)
|
||||||
|
calls = int(rec.get("calls") or 1)
|
||||||
|
totals["input"] += inp
|
||||||
|
totals["cached"] += cached
|
||||||
|
totals["output"] += out
|
||||||
|
totals["calls"] += calls
|
||||||
|
bucket = turns.get(turn_id)
|
||||||
|
if bucket is None:
|
||||||
|
bucket = turns[turn_id] = {
|
||||||
|
"label": (rec.get("label") or turn_id)[:60],
|
||||||
|
"input": 0, "cached": 0, "output": 0, "calls": 0,
|
||||||
|
}
|
||||||
|
order.append(turn_id)
|
||||||
|
bucket["input"] += inp
|
||||||
|
bucket["cached"] += cached
|
||||||
|
bucket["output"] += out
|
||||||
|
bucket["calls"] += calls
|
||||||
|
for bucket in turns.values():
|
||||||
|
bucket["cached"] = min(bucket["cached"], bucket["input"])
|
||||||
|
return totals, [turns[t] for t in order]
|
||||||
|
|
||||||
|
|
||||||
|
def read_db_usage(session_dir):
|
||||||
|
"""Read cumulative usage for the session from state.db.
|
||||||
|
|
||||||
|
Returns a totals dict (input/cached/output/calls) or None when
|
||||||
|
the session has no session_model_usage rows yet.
|
||||||
|
"""
|
||||||
|
rows = _fetch_all(
|
||||||
|
session_dir,
|
||||||
|
"SELECT api_call_count, input_tokens, output_tokens, "
|
||||||
|
"cache_read_tokens, cache_write_tokens "
|
||||||
|
"FROM session_model_usage WHERE session_id = ?",
|
||||||
|
)
|
||||||
|
if not rows:
|
||||||
|
return None
|
||||||
|
totals = {"input": 0, "cached": 0, "output": 0, "calls": 0}
|
||||||
|
for (calls, inp, out, read, write) in rows:
|
||||||
|
# Disjoint Hermes buckets -> Vibe-style inclusive buckets:
|
||||||
|
# input INCLUDES cached; cached counts only cache reads.
|
||||||
|
totals["input"] += (inp or 0) + (read or 0) + (write or 0)
|
||||||
|
totals["cached"] += read or 0
|
||||||
|
totals["output"] += out or 0
|
||||||
|
totals["calls"] += calls or 0
|
||||||
|
return totals
|
||||||
|
|
||||||
|
|
||||||
|
def _ledger_path(session_dir) -> pathlib.Path:
|
||||||
|
return (session_dir.db_path.parent.joinpath(*LEDGER_DIR)
|
||||||
|
/ f"{session_dir.session_id}.jsonl")
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_one(session_dir, query, params=()):
|
||||||
|
db_path = session_dir.db_path
|
||||||
|
if not db_path.is_file():
|
||||||
|
raise FileNotFoundError(f"Hermes state.db not found: {db_path}")
|
||||||
|
with _connect(db_path) as con:
|
||||||
|
return con.execute(query, (session_dir.session_id, *params)).fetchone()
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_all(session_dir, query, params=()):
|
||||||
|
db_path = session_dir.db_path
|
||||||
|
if not db_path.is_file():
|
||||||
|
raise FileNotFoundError(f"Hermes state.db not found: {db_path}")
|
||||||
|
with _connect(db_path) as con:
|
||||||
|
return con.execute(query, (session_dir.session_id, *params)).fetchall()
|
||||||
|
|
||||||
|
|
||||||
|
class HermesHarness(HarnessAdapter):
|
||||||
|
"""Hermes Agent session logs (state.db + plugin ledger)."""
|
||||||
|
|
||||||
|
name = "hermes"
|
||||||
|
home_env_var = "HERMES_HOME"
|
||||||
|
home_dir_name = ".hermes"
|
||||||
|
|
||||||
|
def default_home(self):
|
||||||
|
return default_home()
|
||||||
|
|
||||||
|
def resolve_session_dir(self, session_arg, home):
|
||||||
|
session = resolve_session_dir(session_arg, home)
|
||||||
|
if not session.db_path.is_file():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Hermes state.db not found: {session.db_path}"
|
||||||
|
)
|
||||||
|
if session.session_id is None:
|
||||||
|
session_id = _latest_session_id(session.db_path)
|
||||||
|
session = HermesSession(
|
||||||
|
db_path=session.db_path, session_id=session_id)
|
||||||
|
elif not _session_exists(session.db_path, session.session_id):
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"session not found: {session.session_id} "
|
||||||
|
f"(checked {session.db_path})"
|
||||||
|
)
|
||||||
|
return session
|
||||||
|
|
||||||
|
def read_meta(self, session_dir):
|
||||||
|
title, model_name, alias = read_meta(session_dir)
|
||||||
|
return SessionMeta(title=title, model_name=model_name,
|
||||||
|
alias=alias)
|
||||||
|
|
||||||
|
def collect_usage(self, session_dir):
|
||||||
|
ledger_totals, ledger_turns = parse_ledger(session_dir)
|
||||||
|
db_totals = read_db_usage(session_dir)
|
||||||
|
if db_totals:
|
||||||
|
# The DB's cumulative counter is authoritative (like
|
||||||
|
# Vibe's projection-state): it survives ledger loss.
|
||||||
|
input_tokens = db_totals["input"]
|
||||||
|
cached = db_totals["cached"]
|
||||||
|
output = db_totals["output"]
|
||||||
|
calls = db_totals["calls"]
|
||||||
|
source = "state.db"
|
||||||
|
else:
|
||||||
|
input_tokens = ledger_totals["input"]
|
||||||
|
cached = ledger_totals["cached"]
|
||||||
|
output = ledger_totals["output"]
|
||||||
|
calls = ledger_totals["calls"]
|
||||||
|
source = "ledger"
|
||||||
|
return SessionUsage(
|
||||||
|
input_tokens=input_tokens,
|
||||||
|
cached_input_tokens=min(cached, input_tokens),
|
||||||
|
output_tokens=output,
|
||||||
|
calls=calls,
|
||||||
|
usage_source=source,
|
||||||
|
turns=[
|
||||||
|
TurnUsage(
|
||||||
|
label=t["label"], calls=t["calls"],
|
||||||
|
input_tokens=t["input"],
|
||||||
|
cached_input_tokens=t["cached"],
|
||||||
|
output_tokens=t["output"],
|
||||||
|
)
|
||||||
|
for t in ledger_turns
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _latest_session_id(db_path: pathlib.Path) -> str:
|
||||||
|
"""Most recently active session id (sessions.last_activity_at)."""
|
||||||
|
with _connect(db_path) as con:
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT id FROM sessions "
|
||||||
|
"ORDER BY last_activity_at DESC, started_at DESC LIMIT 1"
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
raise FileNotFoundError(f"no sessions in {db_path}")
|
||||||
|
return row[0]
|
||||||
|
|
||||||
|
|
||||||
|
def _session_exists(db_path: pathlib.Path, session_id: str) -> bool:
|
||||||
|
with _connect(db_path) as con:
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT 1 FROM sessions WHERE id = ?", (session_id,)
|
||||||
|
).fetchone()
|
||||||
|
return row is not None
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
"""Hermes integration package: plugin + installer.
|
||||||
|
|
||||||
|
agent_cost_estimator.integration.hermes.plugin is the Hermes
|
||||||
|
plugin (copied into <hermes-home>/plugins/ by install.py);
|
||||||
|
install.py is a standalone-capable installer.
|
||||||
|
"""
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
"""Hermes integration: plugin directory + installer.
|
||||||
|
|
||||||
|
The plugin/ subdirectory IS the Hermes plugin (plugin.yaml +
|
||||||
|
__init__.py with register(ctx)); install.py copies it into
|
||||||
|
<hermes-home>/plugins/agent-cost-estimator/ and enables it.
|
||||||
|
"""
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Install the agent-cost-estimator Hermes plugin into a Hermes home.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python -m agent_cost_estimator.integration.hermes.install
|
||||||
|
[--hermes-home DIR] [--uninstall] [--dry-run]
|
||||||
|
|
||||||
|
Or, from a source checkout / any machine with the package installed:
|
||||||
|
python path/to/agent-cost-estimator/agent_cost_estimator/integration/hermes/install.py
|
||||||
|
[--hermes-home DIR] [--uninstall] [--dry-run]
|
||||||
|
|
||||||
|
What it does (idempotent):
|
||||||
|
1. Copies the plugin (plugin.yaml + __init__.py) to
|
||||||
|
<home>/plugins/agent-cost-estimator/
|
||||||
|
2. Runs `hermes plugins enable agent-cost-estimator` (via the
|
||||||
|
Hermes CLI when on PATH; otherwise prints the manual step)
|
||||||
|
|
||||||
|
After a restart of Hermes (CLI/TUI/desktop), a line like:
|
||||||
|
|
||||||
|
-- zai:zai-glm-5-3 | session:$0.42 | turn:$0.16
|
||||||
|
|
||||||
|
prints after every assistant turn, mirroring the Mistral Vibe CLI
|
||||||
|
hook. The hermes harness adapter reads usage back via:
|
||||||
|
|
||||||
|
agent-cost-estimator --harness hermes [options]
|
||||||
|
|
||||||
|
Uninstall: --uninstall removes the plugin directory; the enable
|
||||||
|
entry is left in config.yaml (harmless) but can be removed with
|
||||||
|
`hermes plugins disable agent-cost-estimator`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
PLUGIN_NAME = "agent-cost-estimator"
|
||||||
|
PLUGIN_SRC = pathlib.Path(__file__).resolve().parent / "plugin"
|
||||||
|
|
||||||
|
|
||||||
|
def hermes_home(args) -> pathlib.Path:
|
||||||
|
if args.hermes_home:
|
||||||
|
return pathlib.Path(args.hermes_home).expanduser()
|
||||||
|
env = os.environ.get("HERMES_HOME")
|
||||||
|
if env:
|
||||||
|
return pathlib.Path(env).expanduser()
|
||||||
|
return pathlib.Path.home() / ".hermes"
|
||||||
|
|
||||||
|
|
||||||
|
def install(home: pathlib.Path, dry_run: bool = False) -> int:
|
||||||
|
dest = home / "plugins" / PLUGIN_NAME
|
||||||
|
if not PLUGIN_SRC.is_dir():
|
||||||
|
print(f"error: plugin source not found: {PLUGIN_SRC}",
|
||||||
|
file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
if not (home / "state.db").is_file() and not (home / "config.yaml").is_file():
|
||||||
|
print(f"error: {home} does not look like a Hermes home "
|
||||||
|
f"(no state.db or config.yaml); pass --hermes-home DIR",
|
||||||
|
file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
print(f"Installing {PLUGIN_NAME} plugin -> {dest}")
|
||||||
|
if dry_run:
|
||||||
|
print("dry run: no files written")
|
||||||
|
return 0
|
||||||
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
# Preserve an already-tuned installed pricing table across
|
||||||
|
# reinstalls (rate edits live in the installed copy).
|
||||||
|
keep_pricing = None
|
||||||
|
if dest.exists():
|
||||||
|
old_pricing = dest / "pricing.json"
|
||||||
|
if old_pricing.is_file():
|
||||||
|
keep_pricing = old_pricing.read_bytes()
|
||||||
|
shutil.rmtree(dest)
|
||||||
|
shutil.copytree(PLUGIN_SRC, dest)
|
||||||
|
print(f"Installed plugin files: {dest}")
|
||||||
|
# Ship a pricing table with the plugin: the plugin reads
|
||||||
|
# pricing.json next to itself first, independent of the
|
||||||
|
# package being importable in Hermes's interpreter.
|
||||||
|
dest_pricing = dest / "pricing.json"
|
||||||
|
if keep_pricing is not None:
|
||||||
|
dest_pricing.write_bytes(keep_pricing)
|
||||||
|
print(f"Kept existing pricing table: {dest_pricing}")
|
||||||
|
else:
|
||||||
|
pricing_src = (PLUGIN_SRC.parent.parent.parent / "pricing.json")
|
||||||
|
if pricing_src.is_file():
|
||||||
|
shutil.copy2(pricing_src, dest_pricing)
|
||||||
|
print(f"Installed pricing table: {dest_pricing}")
|
||||||
|
else:
|
||||||
|
print(f"warning: bundled pricing.json not found at "
|
||||||
|
f"{pricing_src}; plugin falls back to default rates")
|
||||||
|
# Enable it (plugins are opt-in)
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["hermes", "plugins", "enable", PLUGIN_NAME],
|
||||||
|
capture_output=True, text=True, timeout=60,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
print(f"Enabled plugin: hermes plugins enable {PLUGIN_NAME}")
|
||||||
|
else:
|
||||||
|
print(f"warning: `hermes plugins enable {PLUGIN_NAME}` "
|
||||||
|
f"failed (exit {result.returncode}): "
|
||||||
|
f"{(result.stderr or result.stdout).strip()}")
|
||||||
|
print(f"Enable manually with: hermes plugins enable {PLUGIN_NAME}")
|
||||||
|
except FileNotFoundError:
|
||||||
|
print("warning: `hermes` CLI not on PATH; enable manually with:")
|
||||||
|
print(f" hermes plugins enable {PLUGIN_NAME}")
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
print("warning: `hermes plugins enable` timed out; enable "
|
||||||
|
"manually with:")
|
||||||
|
print(f" hermes plugins enable {PLUGIN_NAME}")
|
||||||
|
print(f"\nDone. Restart Hermes, then every assistant turn ends with:")
|
||||||
|
print(" -- provider:model | session:$X.XX | turn:$Y.YY")
|
||||||
|
print("Query any session with: agent-cost-estimator --harness hermes")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def uninstall(home: pathlib.Path, dry_run: bool = False) -> int:
|
||||||
|
dest = home / "plugins" / PLUGIN_NAME
|
||||||
|
if not dest.exists():
|
||||||
|
print(f"{dest} does not exist; nothing to uninstall")
|
||||||
|
return 0
|
||||||
|
print(f"Removing {dest}")
|
||||||
|
if dry_run:
|
||||||
|
print("dry run: no files removed")
|
||||||
|
return 0
|
||||||
|
shutil.rmtree(dest)
|
||||||
|
print("Removed. Optionally clean the config entry with:")
|
||||||
|
print(f" hermes plugins disable {PLUGIN_NAME}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Install the agent-cost-estimator plugin into "
|
||||||
|
"a Hermes home.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--hermes-home", default=None,
|
||||||
|
help="Hermes profile home (default: $HERMES_HOME or ~/.hermes)")
|
||||||
|
parser.add_argument(
|
||||||
|
"--uninstall", action="store_true",
|
||||||
|
help="remove the plugin instead of installing it")
|
||||||
|
parser.add_argument(
|
||||||
|
"--dry-run", action="store_true",
|
||||||
|
help="show what would change without writing")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
home = hermes_home(args)
|
||||||
|
if args.uninstall:
|
||||||
|
return uninstall(home, dry_run=args.dry_run)
|
||||||
|
return install(home, dry_run=args.dry_run)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,436 @@
|
|||||||
|
"""agent-cost-estimator Hermes plugin.
|
||||||
|
|
||||||
|
Prints a cost line after every assistant turn, in the same shape as
|
||||||
|
the Mistral Vibe CLI hook:
|
||||||
|
|
||||||
|
-- provider:model | session:$X.XX | turn:$Y.YY
|
||||||
|
|
||||||
|
How it works (mirrors the Vibe integration's architecture):
|
||||||
|
|
||||||
|
- post_api_request / post_auxiliary_call: append one JSON line per
|
||||||
|
API call to the session ledger
|
||||||
|
(<home>/plugin-data/agent-cost-estimator/usage/<session_id>.jsonl),
|
||||||
|
the plugin-side equivalent of Vibe's journal files. This is what
|
||||||
|
gives agent-cost-estimator its per-turn breakdown after a
|
||||||
|
restart; live totals for the printed line come from an in-memory
|
||||||
|
accumulator so printing never races the ledger flush.
|
||||||
|
- post_llm_call: once the turn's final response is settled (after
|
||||||
|
_persist_session flushed Hermes's token queue), print the cost
|
||||||
|
line: session total read from state.db's cumulative
|
||||||
|
session_model_usage counter (the authoritative source, like
|
||||||
|
Vibe's projection-state) plus the turn's in-memory bucket.
|
||||||
|
|
||||||
|
Subagents (platform == "subagent") and cron runs record to the
|
||||||
|
ledger but never print. One-shot runs (hermes chat -q) neither
|
||||||
|
print nor record - their stdout must stay machine-readable.
|
||||||
|
|
||||||
|
Stdlib only. Every callback fails soft: a cost line must never
|
||||||
|
break a turn.
|
||||||
|
|
||||||
|
Known limitation: Hermes skips a timeout-bounded hook callback
|
||||||
|
that is still running for the same turn_id, so under MoA advisor
|
||||||
|
fan-out a concurrently-fired duplicate call may not be ledgered.
|
||||||
|
Session totals are unaffected (state.db is authoritative); only
|
||||||
|
that call's contribution to the per-turn breakdown can be lost.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import queue
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Optional, Tuple
|
||||||
|
|
||||||
|
logger = logging.getLogger("agent_cost_estimator.hermes_plugin")
|
||||||
|
|
||||||
|
DB_NAME = "state.db"
|
||||||
|
LEDGER_DIR = Path("plugin-data") / "agent-cost-estimator" / "usage"
|
||||||
|
LINE_PREFIX = "-- "
|
||||||
|
PRINT_PLATFORM = "cli"
|
||||||
|
|
||||||
|
# ── ledger (durable per-call record, Vibe-journal equivalent) ────────
|
||||||
|
|
||||||
|
# Records are queued and appended by a daemon drainer so the hook
|
||||||
|
# callback never does I/O.
|
||||||
|
_LEDGER_QUEUE: "queue.Queue[dict]" = queue.Queue(maxsize=10_000)
|
||||||
|
_DRAINER_STARTED = False
|
||||||
|
_DRAINER_LOCK = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _hermes_home() -> Path:
|
||||||
|
env = os.environ.get("HERMES_HOME")
|
||||||
|
return Path(env).expanduser() if env else Path.home() / ".hermes"
|
||||||
|
|
||||||
|
|
||||||
|
def _ledger_path(session_id: str) -> Path:
|
||||||
|
return _hermes_home() / LEDGER_DIR / f"{session_id}.jsonl"
|
||||||
|
|
||||||
|
|
||||||
|
def _drain_ledger() -> None:
|
||||||
|
"""Append queued records; the only ledger I/O thread."""
|
||||||
|
while True:
|
||||||
|
record = _LEDGER_QUEUE.get()
|
||||||
|
try:
|
||||||
|
path = _ledger_path(str(record.get("session_id") or ""))
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
line = json.dumps(record, separators=(",", ":"))
|
||||||
|
with path.open("a", encoding="utf-8") as handle:
|
||||||
|
handle.write(line + "\n")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("cost ledger append failed: %s", exc)
|
||||||
|
finally:
|
||||||
|
_LEDGER_QUEUE.task_done()
|
||||||
|
|
||||||
|
|
||||||
|
def _start_drainer() -> None:
|
||||||
|
"""Start the single ledger drainer thread (idempotent)."""
|
||||||
|
global _DRAINER_STARTED
|
||||||
|
if _DRAINER_STARTED:
|
||||||
|
return
|
||||||
|
with _DRAINER_LOCK:
|
||||||
|
if _DRAINER_STARTED:
|
||||||
|
return
|
||||||
|
threading.Thread(
|
||||||
|
target=_drain_ledger,
|
||||||
|
name="agent-cost-estimator-ledger",
|
||||||
|
daemon=True,
|
||||||
|
).start()
|
||||||
|
_DRAINER_STARTED = True
|
||||||
|
|
||||||
|
|
||||||
|
# ── in-memory per-turn accumulator (exact at print time) ─────────────
|
||||||
|
|
||||||
|
# (session_id, turn_id) -> {input, cached, output}; the printed turn
|
||||||
|
# cost reads this, so it never races the ledger drainer.
|
||||||
|
_turn_usage: Dict[Tuple[str, str], Dict[str, int]] = {}
|
||||||
|
_usage_lock = threading.Lock()
|
||||||
|
|
||||||
|
# turn_id -> user-message head, labels the ledger's turn bucket.
|
||||||
|
_turn_labels: Dict[str, str] = {}
|
||||||
|
_label_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _usage_bucket(usage: Optional[dict]) -> Optional[dict]:
|
||||||
|
"""Map a hook usage summary (CanonicalUsage keys) to ledger
|
||||||
|
buckets: input (incl. cached), cached, output.
|
||||||
|
|
||||||
|
CanonicalUsage buckets are DISJOINT: input_tokens EXCLUDES the
|
||||||
|
cache buckets. agent-cost-estimator's SessionUsage contract is
|
||||||
|
Vibe-style (input INCLUDES cached), so fold here:
|
||||||
|
input = input + cache_read + cache_write (= prompt_tokens),
|
||||||
|
cached = cache_read, fresh = input - cached = input + write.
|
||||||
|
"""
|
||||||
|
if not isinstance(usage, dict):
|
||||||
|
return None
|
||||||
|
inp = int(usage.get("input_tokens") or 0)
|
||||||
|
read = int(usage.get("cache_read_tokens") or 0)
|
||||||
|
write = int(usage.get("cache_write_tokens") or 0)
|
||||||
|
out = int(usage.get("output_tokens") or 0)
|
||||||
|
if not (inp or read or write or out):
|
||||||
|
return None # usage-less call: nothing to price or record
|
||||||
|
return {
|
||||||
|
"input_tokens": inp + read + write, # prompt_tokens
|
||||||
|
"cached_input_tokens": read,
|
||||||
|
"output_tokens": out,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _record_call(
|
||||||
|
*, session_id: str, turn_id: str, model: str, provider: str,
|
||||||
|
usage: Optional[dict], aux_task: Optional[str] = None,
|
||||||
|
api_request_id: Optional[str] = None,
|
||||||
|
) -> None:
|
||||||
|
"""Ledger one API call and fold it into the in-memory bucket."""
|
||||||
|
bucket = _usage_bucket(usage)
|
||||||
|
if bucket is None:
|
||||||
|
return # usage-less call: nothing to price
|
||||||
|
with _usage_lock:
|
||||||
|
key = (session_id, turn_id)
|
||||||
|
acc = _turn_usage.setdefault(
|
||||||
|
key, {"input": 0, "cached": 0, "output": 0})
|
||||||
|
acc["input"] += bucket["input_tokens"]
|
||||||
|
acc["cached"] += bucket["cached_input_tokens"]
|
||||||
|
acc["output"] += bucket["output_tokens"]
|
||||||
|
# Bound the map: keep the most recent 256 turn buckets.
|
||||||
|
if len(_turn_usage) > 512:
|
||||||
|
for stale in list(_turn_usage)[:-512]:
|
||||||
|
_turn_usage.pop(stale, None)
|
||||||
|
with _label_lock:
|
||||||
|
label = _turn_labels.get(turn_id)
|
||||||
|
_append_ledger({
|
||||||
|
"ts": time.time(),
|
||||||
|
"session_id": session_id,
|
||||||
|
"turn_id": turn_id,
|
||||||
|
"model": model,
|
||||||
|
"provider": provider,
|
||||||
|
"api_request_id": api_request_id,
|
||||||
|
"aux_task": aux_task,
|
||||||
|
"label": label,
|
||||||
|
"calls": 1,
|
||||||
|
**bucket,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _append_ledger(record: dict) -> None:
|
||||||
|
"""Queue one call record; never raises into the hook."""
|
||||||
|
try:
|
||||||
|
_start_drainer()
|
||||||
|
_LEDGER_QUEUE.put_nowait(record)
|
||||||
|
except queue.Full:
|
||||||
|
# Drop rather than block the agent loop; session totals
|
||||||
|
# still come from state.db, so only this call's turn
|
||||||
|
# breakdown is lost.
|
||||||
|
logger.debug("cost ledger queue full; record dropped")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("cost ledger enqueue failed: %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
# ── hooks ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _on_pre_llm_call(**kwargs: Any) -> None:
|
||||||
|
"""Remember the user-message head as this turn's label."""
|
||||||
|
try:
|
||||||
|
turn_id = str(kwargs.get("turn_id") or "")
|
||||||
|
message = kwargs.get("user_message")
|
||||||
|
if not turn_id or not isinstance(message, str):
|
||||||
|
return
|
||||||
|
if not message.strip():
|
||||||
|
return
|
||||||
|
with _label_lock:
|
||||||
|
# pre_llm_call fires once per turn; earliest label wins.
|
||||||
|
_turn_labels.setdefault(turn_id, " ".join(message.split())[:60])
|
||||||
|
if len(_turn_labels) > 128:
|
||||||
|
for stale in list(_turn_labels)[:-128]:
|
||||||
|
_turn_labels.pop(stale, None)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("pre_llm_call label capture failed: %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
def _on_post_api_request(**kwargs: Any) -> None:
|
||||||
|
"""Ledger one main-loop API call."""
|
||||||
|
try:
|
||||||
|
session_id = str(kwargs.get("session_id") or "")
|
||||||
|
turn_id = str(kwargs.get("turn_id") or "(no turn)")
|
||||||
|
if not session_id:
|
||||||
|
return
|
||||||
|
_record_call(
|
||||||
|
session_id=session_id,
|
||||||
|
turn_id=turn_id,
|
||||||
|
model=str(kwargs.get("model") or ""),
|
||||||
|
provider=str(kwargs.get("provider") or ""),
|
||||||
|
usage=kwargs.get("usage"),
|
||||||
|
api_request_id=str(kwargs.get("api_request_id") or "") or None,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("post_api_request record failed: %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
def _on_post_auxiliary_call(**kwargs: Any) -> None:
|
||||||
|
"""Ledger one auxiliary LLM call (titling, compression, MoA...).
|
||||||
|
|
||||||
|
Auxiliary spend is real and part of session totals; it folds
|
||||||
|
into the parent turn's bucket and never prints a line of its
|
||||||
|
own.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
session_id = str(kwargs.get("session_id") or "")
|
||||||
|
if not session_id:
|
||||||
|
return # turn-less caller (cron, gateway idle work)
|
||||||
|
turn_id = str(kwargs.get("turn_id") or "(aux)")
|
||||||
|
_record_call(
|
||||||
|
session_id=session_id,
|
||||||
|
turn_id=turn_id,
|
||||||
|
model=str(kwargs.get("model") or ""),
|
||||||
|
provider=str(kwargs.get("provider") or ""),
|
||||||
|
usage=kwargs.get("usage"),
|
||||||
|
aux_task=str(kwargs.get("aux_task") or "") or None,
|
||||||
|
api_request_id=str(kwargs.get("api_request_id") or "") or None,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("post_auxiliary_call record failed: %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
# ── turn-end cost line ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _db_session_totals(session_id: str) -> Optional[dict]:
|
||||||
|
"""Cumulative tokens for the session from state.db (read-only).
|
||||||
|
|
||||||
|
Mirrors agent_cost_estimator.harnesses.hermes.read_db_usage.
|
||||||
|
Returns None when there are no usage rows yet.
|
||||||
|
"""
|
||||||
|
db_path = _hermes_home() / DB_NAME
|
||||||
|
if not db_path.is_file():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5.0)
|
||||||
|
except sqlite3.Error:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT api_call_count, input_tokens, output_tokens, "
|
||||||
|
"cache_read_tokens, cache_write_tokens "
|
||||||
|
"FROM session_model_usage WHERE session_id = ?",
|
||||||
|
(session_id,),
|
||||||
|
).fetchall()
|
||||||
|
except sqlite3.Error:
|
||||||
|
return None
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
if not rows:
|
||||||
|
return None
|
||||||
|
totals = {"input": 0, "cached": 0, "output": 0, "calls": 0}
|
||||||
|
for (calls, inp, out, read, write) in rows:
|
||||||
|
# Disjoint Hermes buckets -> Vibe-style inclusive buckets.
|
||||||
|
totals["input"] += (inp or 0) + (read or 0) + (write or 0)
|
||||||
|
totals["cached"] += read or 0
|
||||||
|
totals["output"] += out or 0
|
||||||
|
totals["calls"] += calls or 0
|
||||||
|
return totals
|
||||||
|
|
||||||
|
|
||||||
|
def _turn_bucket(session_id: str, turn_id: str) -> Dict[str, int]:
|
||||||
|
"""This turn's in-memory bucket (exact; no ledger read)."""
|
||||||
|
with _usage_lock:
|
||||||
|
return dict(_turn_usage.get((session_id, turn_id))
|
||||||
|
or {"input": 0, "cached": 0, "output": 0})
|
||||||
|
|
||||||
|
|
||||||
|
def _load_local_pricing(path: pathlib.Path) -> Optional[dict]:
|
||||||
|
"""Stdlib-only loader/validator for a plugin-local pricing.json.
|
||||||
|
|
||||||
|
Returns the parsed pricing dict or None. Accepts the same
|
||||||
|
provider-based format as agent_cost_estimator.pricing.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
data = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return None
|
||||||
|
if not isinstance(data, dict) or not isinstance(
|
||||||
|
data.get("providers"), dict):
|
||||||
|
return None
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _lookup_local(pricing: dict, model: str) -> Optional[Tuple[dict, str, bool]]:
|
||||||
|
"""Find (rates, key, is_default) for the model in a local table.
|
||||||
|
|
||||||
|
Same lookup order as agent_cost_estimator.pricing: exact model
|
||||||
|
match across providers, then the first provider's default.
|
||||||
|
"""
|
||||||
|
providers = pricing.get("providers") or {}
|
||||||
|
for prov_name, prov in providers.items():
|
||||||
|
models = prov.get("models") or {}
|
||||||
|
if model in models:
|
||||||
|
rates = dict(models[model])
|
||||||
|
if all(rates.get(k) is not None for k in
|
||||||
|
("input_price", "cached_input_price", "output_price")):
|
||||||
|
return rates, f"{prov_name}:{model}", False
|
||||||
|
for prov_name, prov in providers.items():
|
||||||
|
models = prov.get("models") or {}
|
||||||
|
if "default" in models:
|
||||||
|
rates = dict(models["default"])
|
||||||
|
if all(rates.get(k) is not None for k in
|
||||||
|
("input_price", "cached_input_price", "output_price")):
|
||||||
|
return rates, f"{prov_name}:default", True
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _pricing(model: str) -> Tuple[dict, str, bool]:
|
||||||
|
"""Look up rates for the model.
|
||||||
|
|
||||||
|
Source order:
|
||||||
|
1. pricing.json next to this plugin (installed by install.py;
|
||||||
|
editable to tune rates without touching the package; read
|
||||||
|
with a stdlib-only loader so the plugin works even where
|
||||||
|
the agent_cost_estimator package is not installed)
|
||||||
|
2. the agent_cost_estimator package's bundled pricing.json
|
||||||
|
(when the package is importable in this interpreter)
|
||||||
|
3. hard-coded global default rates
|
||||||
|
|
||||||
|
Returns (rates, provider:model key, is_default).
|
||||||
|
"""
|
||||||
|
# 1. plugin-local pricing.json
|
||||||
|
local = pathlib.Path(__file__).resolve().parent / "pricing.json"
|
||||||
|
pricing = _load_local_pricing(local)
|
||||||
|
if pricing is not None:
|
||||||
|
found = _lookup_local(pricing, model)
|
||||||
|
if found is not None:
|
||||||
|
return found
|
||||||
|
logger.debug("model %r not in plugin-local pricing.json", model)
|
||||||
|
# 2. the installed package's table
|
||||||
|
try:
|
||||||
|
from agent_cost_estimator.pricing import (
|
||||||
|
DEFAULT_PRICING_PATH, get_model_prices, load_pricing,
|
||||||
|
)
|
||||||
|
pricing = load_pricing(DEFAULT_PRICING_PATH)
|
||||||
|
rates, key, _currency, is_default = get_model_prices(pricing, model, None)
|
||||||
|
return rates, key, bool(is_default)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("pricing lookup failed (%s); using default", exc)
|
||||||
|
# 3. hard-coded fallback (mirrors the bundled default rates)
|
||||||
|
return (
|
||||||
|
{"input_price": 1.5, "cached_input_price": 0.15,
|
||||||
|
"output_price": 7.5},
|
||||||
|
"default:default",
|
||||||
|
True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _cost(tokens: dict, rates: dict) -> float:
|
||||||
|
"""USD cost of one bucket at per-million rates."""
|
||||||
|
fresh = max(0, tokens["input"] - tokens["cached"])
|
||||||
|
return (
|
||||||
|
fresh * rates["input_price"]
|
||||||
|
+ tokens["cached"] * rates["cached_input_price"]
|
||||||
|
+ tokens["output"] * rates["output_price"]
|
||||||
|
) / 1_000_000
|
||||||
|
|
||||||
|
|
||||||
|
def _print_cost_line(session_id: str, turn_id: str, model: str) -> None:
|
||||||
|
"""Compute and print the turn's cost line."""
|
||||||
|
session = _db_session_totals(session_id)
|
||||||
|
if session is None:
|
||||||
|
return # nothing recorded yet for this session
|
||||||
|
turn = _turn_bucket(session_id, turn_id)
|
||||||
|
rates, key, is_default = _pricing(model)
|
||||||
|
session_usd = _cost(session, rates)
|
||||||
|
turn_usd = _cost(turn, rates)
|
||||||
|
line = (
|
||||||
|
f"{LINE_PREFIX}{key} | session:${session_usd:.2f} | "
|
||||||
|
f"turn:${turn_usd:.2f}"
|
||||||
|
)
|
||||||
|
if is_default:
|
||||||
|
line += "\n **model price not available**"
|
||||||
|
print(line, flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _on_post_llm_call(**kwargs: Any) -> None:
|
||||||
|
"""Turn end: print the cost line for the session a human reads."""
|
||||||
|
try:
|
||||||
|
platform = str(kwargs.get("platform") or "")
|
||||||
|
if platform != PRINT_PLATFORM:
|
||||||
|
return # subagents, cron, gateway platforms: record only
|
||||||
|
if os.environ.get("HERMES_SINGLE_QUERY_SESSION") == "1":
|
||||||
|
return # hermes chat -q: stdout must stay machine-readable
|
||||||
|
session_id = str(kwargs.get("session_id") or "")
|
||||||
|
turn_id = str(kwargs.get("turn_id") or "")
|
||||||
|
if not session_id or not turn_id:
|
||||||
|
return
|
||||||
|
_print_cost_line(session_id, turn_id,
|
||||||
|
str(kwargs.get("model") or ""))
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("post_llm_call cost line failed: %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
def register(ctx) -> None:
|
||||||
|
"""Register the cost-tracker hooks."""
|
||||||
|
ctx.register_hook("pre_llm_call", _on_pre_llm_call)
|
||||||
|
ctx.register_hook("post_api_request", _on_post_api_request)
|
||||||
|
ctx.register_hook("post_auxiliary_call", _on_post_auxiliary_call)
|
||||||
|
ctx.register_hook("post_llm_call", _on_post_llm_call)
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
name: agent-cost-estimator
|
||||||
|
version: 0.4.0
|
||||||
|
description: "Print an estimated cost line (provider:model | session:$X | turn:$Y) after every assistant turn, recorded from Hermes session usage."
|
||||||
|
author: "nvaldez"
|
||||||
|
provides_hooks:
|
||||||
|
- pre_llm_call
|
||||||
|
- post_api_request
|
||||||
|
- post_auxiliary_call
|
||||||
|
- post_llm_call
|
||||||
@@ -8,7 +8,8 @@ description: Estimate USD cost of AI agent sessions; run "agent-cost-estimator -
|
|||||||
|
|
||||||
Report token usage and an estimated USD cost for an AI agent
|
Report token usage and an estimated USD cost for an AI agent
|
||||||
session, read from the harness's local logs. Ships with the
|
session, read from the harness's local logs. Ships with the
|
||||||
`vibe` harness; other harnesses plug in as adapters.
|
`vibe` (Mistral Vibe) and `hermes` (Hermes Agent) harnesses;
|
||||||
|
other harnesses plug in as adapters.
|
||||||
|
|
||||||
Run `agent-cost-estimator --skill` at any time to re-read these
|
Run `agent-cost-estimator --skill` at any time to re-read these
|
||||||
instructions. Everything below assumes the installed
|
instructions. Everything below assumes the installed
|
||||||
@@ -27,7 +28,10 @@ agent-cost-estimator -s -> 2.1148 (bare session cost)
|
|||||||
agent-cost-estimator -t -> 2.4979 (bare latest-turn cost)
|
agent-cost-estimator -t -> 2.4979 (bare latest-turn cost)
|
||||||
agent-cost-estimator -d -> full report table
|
agent-cost-estimator -d -> full report table
|
||||||
agent-cost-estimator --json -> one JSON object, all numbers
|
agent-cost-estimator --json -> one JSON object, all numbers
|
||||||
agent-cost-estimator --integrate -> install Vibe hooks automatically
|
agent-cost-estimator --harness hermes
|
||||||
|
-> same outputs for a Hermes Agent session (state.db)
|
||||||
|
agent-cost-estimator --integrate mistral -> install Vibe hooks
|
||||||
|
agent-cost-estimator --integrate hermes -> install the Hermes plugin
|
||||||
```
|
```
|
||||||
|
|
||||||
The default line answers "what does this session cost and what
|
The default line answers "what does this session cost and what
|
||||||
@@ -36,18 +40,27 @@ scripting; `-d` shows the table with token counts, pricing
|
|||||||
source, and caveats; `--json` is the full machine-readable
|
source, and caveats; `--json` is the full machine-readable
|
||||||
report (tokens, cost breakdown, per-turn list, pricing).
|
report (tokens, cost breakdown, per-turn list, pricing).
|
||||||
|
|
||||||
Use `--integrate` to set up Vibe CLI hooks automatically on any
|
Use `--integrate mistral` to set up Vibe CLI hooks automatically
|
||||||
machine. This installs the hook script, configuration, and updates
|
on any machine (hook script, configuration, bash allowlist).
|
||||||
the bash allowlist.
|
Use `--integrate hermes` to install the Hermes Agent plugin: it
|
||||||
|
copies the plugin into `<hermes-home>/plugins/`, enables it, and
|
||||||
|
after a Hermes restart prints `-- provider:model | session:$X |
|
||||||
|
turn:$Y` after every assistant turn, mirroring the Vibe hook.
|
||||||
|
`--rm-integrate hermes` removes it.
|
||||||
|
|
||||||
## What to trust
|
## What to trust
|
||||||
|
|
||||||
- Session-wide token totals come from the harness's cumulative
|
- Session-wide token totals come from the harness's cumulative
|
||||||
counter (for vibe: `projection-state.json`), which survives
|
counter (vibe: `projection-state.json`; hermes: the
|
||||||
|
`session_model_usage` table in `state.db`), which survives
|
||||||
log rotation.
|
log rotation.
|
||||||
- Call counts and per-turn breakdowns come from journal files,
|
- Call counts and per-turn breakdowns come from journal files
|
||||||
which the harness prunes: they cover only the window on disk.
|
(vibe) or the plugin's usage ledger (hermes:
|
||||||
The latest turn is always in that window, so `-t` is safe.
|
`<home>/plugin-data/agent-cost-estimator/usage/*.jsonl`),
|
||||||
|
which the harness may outlive: they cover only what is on
|
||||||
|
disk. The latest turn is always there, so `-t` is safe.
|
||||||
|
For hermes, the ledger only exists once the plugin is
|
||||||
|
installed; without it, per-turn reads are zero.
|
||||||
- All costs are **estimates** at the configured rates, never
|
- All costs are **estimates** at the configured rates, never
|
||||||
billed figures. Tell the user that, and point them at their
|
billed figures. Tell the user that, and point them at their
|
||||||
provider's console (Mistral: admin.mistral.ai/subscription)
|
provider's console (Mistral: admin.mistral.ai/subscription)
|
||||||
@@ -56,14 +69,17 @@ the bash allowlist.
|
|||||||
## Selecting what to measure
|
## Selecting what to measure
|
||||||
|
|
||||||
```text
|
```text
|
||||||
--harness NAME adapter to read (default: vibe; list: --list-harnesses)
|
--harness NAME adapter to read (default: vibe; also:
|
||||||
|
hermes; list: --list-harnesses)
|
||||||
--session-id ID|PATH a specific session id or directory
|
--session-id ID|PATH a specific session id or directory
|
||||||
--home PATH the harness's data root (default: harness-specific,
|
--home PATH the harness's data root (default: harness-specific,
|
||||||
vibe: $VIBE_HOME or ~/.vibe)
|
vibe: $VIBE_HOME or ~/.vibe;
|
||||||
|
hermes: $HERMES_HOME or ~/.hermes)
|
||||||
```
|
```
|
||||||
|
|
||||||
With no `--session-id`, the most recently active session is
|
With no `--session-id`, the most recently active session is
|
||||||
picked (newest journal file). To inspect another session, pass
|
picked (vibe: newest journal file; hermes: newest
|
||||||
|
`last_activity_at` in state.db). To inspect another session, pass
|
||||||
its id or path.
|
its id or path.
|
||||||
|
|
||||||
## Pricing
|
## Pricing
|
||||||
|
|||||||
+3
-2
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "agent-cost-estimator"
|
name = "agent-cost-estimator"
|
||||||
version = "0.3.0"
|
version = "0.4.0"
|
||||||
description = "Report token usage and estimated USD cost of AI agent sessions from local harness logs"
|
description = "Report token usage and estimated USD cost of AI agent sessions from local harness logs"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.9"
|
requires-python = ">=3.9"
|
||||||
@@ -31,6 +31,7 @@ agent-cost-estimator = "agent_cost_estimator.cli:main"
|
|||||||
|
|
||||||
[project.entry-points."agent_cost_estimator.harnesses"]
|
[project.entry-points."agent_cost_estimator.harnesses"]
|
||||||
vibe = "agent_cost_estimator.harnesses.vibe:VibeHarness"
|
vibe = "agent_cost_estimator.harnesses.vibe:VibeHarness"
|
||||||
|
hermes = "agent_cost_estimator.harnesses.hermes:HermesHarness"
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
dev = [
|
dev = [
|
||||||
@@ -39,7 +40,7 @@ dev = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[tool.setuptools]
|
[tool.setuptools]
|
||||||
packages = ["agent_cost_estimator", "agent_cost_estimator.harnesses", "agent_cost_estimator.skill", "agent_cost_estimator.hooks"]
|
packages = ["agent_cost_estimator", "agent_cost_estimator.harnesses", "agent_cost_estimator.skill", "agent_cost_estimator.hooks", "agent_cost_estimator.integration", "agent_cost_estimator.integration.hermes"]
|
||||||
include-package-data = true
|
include-package-data = true
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ harness registry, and the CLI (pricing flags, -t, --json,
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
@@ -544,3 +545,380 @@ def test_cli_integrate_unsupported_harness(capsys):
|
|||||||
code = cli.main(["--integrate", "unsupported-harness"])
|
code = cli.main(["--integrate", "unsupported-harness"])
|
||||||
assert code == 1
|
assert code == 1
|
||||||
assert "unsupported harness" in capsys.readouterr().err
|
assert "unsupported harness" in capsys.readouterr().err
|
||||||
|
|
||||||
|
|
||||||
|
# --- hermes harness adapter ---
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
from agent_cost_estimator.harnesses import hermes as hh
|
||||||
|
|
||||||
|
|
||||||
|
def make_hermes_home(tmp_path, sessions=(), usage=()):
|
||||||
|
"""Create a fake Hermes home: state.db + optional rows.
|
||||||
|
|
||||||
|
sessions: (id, title, model, last_activity_at)
|
||||||
|
usage: (session_id, model, calls, input, output, read, write)
|
||||||
|
"""
|
||||||
|
home = tmp_path / "hermes-home"
|
||||||
|
home.mkdir()
|
||||||
|
con = sqlite3.connect(home / "state.db")
|
||||||
|
con.execute(
|
||||||
|
"CREATE TABLE sessions (id TEXT PRIMARY KEY, title TEXT, "
|
||||||
|
"model TEXT, started_at REAL, last_activity_at REAL)")
|
||||||
|
con.execute(
|
||||||
|
"CREATE TABLE session_model_usage (session_id TEXT, model TEXT, "
|
||||||
|
"task TEXT, api_call_count INTEGER, input_tokens INTEGER, "
|
||||||
|
"output_tokens INTEGER, cache_read_tokens INTEGER, "
|
||||||
|
"cache_write_tokens INTEGER, reasoning_tokens INTEGER)")
|
||||||
|
for row in sessions:
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO sessions VALUES (?, ?, ?, 1.0, ?)", row)
|
||||||
|
for row in usage:
|
||||||
|
con.execute("INSERT INTO session_model_usage VALUES "
|
||||||
|
"(?, ?, '', ?, ?, ?, ?, ?, 0)", row)
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
return home
|
||||||
|
|
||||||
|
|
||||||
|
def write_hermes_ledger(home, session_id, records):
|
||||||
|
"""Write plugin-style ledger lines for a session."""
|
||||||
|
ledger = (home / "plugin-data" / "agent-cost-estimator" / "usage")
|
||||||
|
ledger.mkdir(parents=True, exist_ok=True)
|
||||||
|
path = ledger / f"{session_id}.jsonl"
|
||||||
|
with path.open("a", encoding="utf-8") as handle:
|
||||||
|
for rec in records:
|
||||||
|
handle.write(json.dumps(rec) + "\n")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_harness_registered():
|
||||||
|
"""The hermes harness is in the registry."""
|
||||||
|
assert "hermes" in available_harnesses()
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_resolves_latest_session(tmp_path):
|
||||||
|
"""No session id picks the most recently active session."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[
|
||||||
|
("s-old", "old", "m", 1.0),
|
||||||
|
("s-new", "new", "m", 2.0),
|
||||||
|
])
|
||||||
|
h = hh.HermesHarness()
|
||||||
|
session = h.resolve_session_dir(None, home)
|
||||||
|
assert session.session_id == "s-new"
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_resolves_explicit_session(tmp_path):
|
||||||
|
"""An explicit session id resolves and unknown ids error."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[("s1", "t", "m", 1.0)])
|
||||||
|
h = hh.HermesHarness()
|
||||||
|
session = h.resolve_session_dir("s1", home)
|
||||||
|
assert session.session_id == "s1"
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
h.resolve_session_dir("missing", home)
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_missing_db_errors(tmp_path):
|
||||||
|
"""A home without state.db is a clear error."""
|
||||||
|
h = hh.HermesHarness()
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
h.resolve_session_dir(None, tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_read_meta(tmp_path):
|
||||||
|
"""read_meta returns title and model from the sessions row."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[
|
||||||
|
("s1", "my session", "zai-glm-5-3", 1.0)])
|
||||||
|
h = hh.HermesHarness()
|
||||||
|
session = h.resolve_session_dir("s1", home)
|
||||||
|
meta = h.read_meta(session)
|
||||||
|
assert meta.title == "my session"
|
||||||
|
assert meta.model_name == "zai-glm-5-3"
|
||||||
|
assert meta.alias is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_collect_usage_db_authoritative(tmp_path):
|
||||||
|
"""state.db cumulative counters win over the ledger for totals."""
|
||||||
|
home = make_hermes_home(
|
||||||
|
tmp_path,
|
||||||
|
sessions=[("s1", "t", "zai-glm-5-3", 1.0)],
|
||||||
|
usage=[("s1", "zai-glm-5-3", 3, 100000, 20000, 900000, 5000)],
|
||||||
|
)
|
||||||
|
# Ledger disagrees (older window); DB must win.
|
||||||
|
write_hermes_ledger(home, "s1", [
|
||||||
|
{"turn_id": "t1", "input_tokens": 10, "cached_input_tokens": 0,
|
||||||
|
"output_tokens": 1, "calls": 1, "label": "x"},
|
||||||
|
])
|
||||||
|
h = hh.HermesHarness()
|
||||||
|
session = h.resolve_session_dir("s1", home)
|
||||||
|
usage = h.collect_usage(session)
|
||||||
|
# input = fresh 100000 + read 900000 + write 5000
|
||||||
|
assert usage.input_tokens == 1005000
|
||||||
|
assert usage.cached_input_tokens == 900000
|
||||||
|
assert usage.output_tokens == 20000
|
||||||
|
assert usage.calls == 3
|
||||||
|
assert usage.usage_source == "state.db"
|
||||||
|
# Per-turn breakdown still comes from the ledger
|
||||||
|
assert len(usage.turns) == 1
|
||||||
|
assert usage.turns[0].label == "x"
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_collect_usage_ledger_fallback(tmp_path):
|
||||||
|
"""Without DB rows (plugin-only), the ledger is the source."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[("s1", "t", "m", 1.0)])
|
||||||
|
write_hermes_ledger(home, "s1", [
|
||||||
|
{"turn_id": "t1", "input_tokens": 100, "cached_input_tokens": 40,
|
||||||
|
"output_tokens": 10, "calls": 1, "label": "first"},
|
||||||
|
{"turn_id": "t1", "input_tokens": 60, "cached_input_tokens": 10,
|
||||||
|
"output_tokens": 5, "calls": 1, "label": "first"},
|
||||||
|
{"turn_id": "t2", "input_tokens": 30, "cached_input_tokens": 30,
|
||||||
|
"output_tokens": 2, "calls": 1, "label": "second"},
|
||||||
|
])
|
||||||
|
h = hh.HermesHarness()
|
||||||
|
session = h.resolve_session_dir("s1", home)
|
||||||
|
usage = h.collect_usage(session)
|
||||||
|
assert usage.usage_source == "ledger"
|
||||||
|
assert usage.input_tokens == 190
|
||||||
|
assert usage.cached_input_tokens == 80
|
||||||
|
assert usage.output_tokens == 17
|
||||||
|
assert usage.calls == 3
|
||||||
|
assert [t.label for t in usage.turns] == ["first", "second"]
|
||||||
|
assert usage.turns[0].input_tokens == 160
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_ledger_dedupes_api_request_id(tmp_path):
|
||||||
|
"""Duplicate api_request_id records count once."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[("s1", "t", "m", 1.0)])
|
||||||
|
write_hermes_ledger(home, "s1", [
|
||||||
|
{"turn_id": "t1", "api_request_id": "r1", "input_tokens": 100,
|
||||||
|
"cached_input_tokens": 0, "output_tokens": 10, "calls": 1},
|
||||||
|
{"turn_id": "t1", "api_request_id": "r1", "input_tokens": 100,
|
||||||
|
"cached_input_tokens": 0, "output_tokens": 10, "calls": 1},
|
||||||
|
{"turn_id": "t1", "api_request_id": "r2", "input_tokens": 50,
|
||||||
|
"cached_input_tokens": 0, "output_tokens": 5, "calls": 1},
|
||||||
|
])
|
||||||
|
h = hh.HermesHarness()
|
||||||
|
session = h.resolve_session_dir("s1", home)
|
||||||
|
usage = h.collect_usage(session)
|
||||||
|
assert usage.calls == 2
|
||||||
|
assert usage.input_tokens == 150
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_ledger_clamps_cached(tmp_path):
|
||||||
|
"""Cached input is clamped to input (defensive)."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[("s1", "t", "m", 1.0)])
|
||||||
|
write_hermes_ledger(home, "s1", [
|
||||||
|
{"turn_id": "t1", "input_tokens": 10,
|
||||||
|
"cached_input_tokens": 999, "output_tokens": 1, "calls": 1},
|
||||||
|
])
|
||||||
|
h = hh.HermesHarness()
|
||||||
|
session = h.resolve_session_dir("s1", home)
|
||||||
|
usage = h.collect_usage(session)
|
||||||
|
assert usage.cached_input_tokens == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_hermes_cli_json(tmp_path, capsys):
|
||||||
|
"""--harness hermes --json reports the session end to end."""
|
||||||
|
home = make_hermes_home(
|
||||||
|
tmp_path,
|
||||||
|
sessions=[("s1", "t", "zai-glm-5-3", 1.0)],
|
||||||
|
usage=[("s1", "zai-glm-5-3", 2, 1000, 500, 5000, 0)],
|
||||||
|
)
|
||||||
|
code = cli.main(["--harness", "hermes", "--home", str(home),
|
||||||
|
"--json"])
|
||||||
|
assert code == 0
|
||||||
|
data = json.loads(capsys.readouterr().out)
|
||||||
|
assert data["harness"] == "hermes"
|
||||||
|
assert data["session_id"] == "s1"
|
||||||
|
assert data["model"] == "zai-glm-5-3"
|
||||||
|
assert data["input_tokens"] == 6000
|
||||||
|
assert data["cached_input_tokens"] == 5000
|
||||||
|
assert data["usage_source"] == "state.db"
|
||||||
|
|
||||||
|
|
||||||
|
# --- hermes plugin (integration) ---
|
||||||
|
|
||||||
|
from agent_cost_estimator.integration.hermes import plugin as hp
|
||||||
|
|
||||||
|
|
||||||
|
class _Ctx:
|
||||||
|
def __init__(self):
|
||||||
|
self.hooks = []
|
||||||
|
|
||||||
|
def register_hook(self, name, cb):
|
||||||
|
self.hooks.append((name, cb))
|
||||||
|
|
||||||
|
|
||||||
|
def test_plugin_registers_all_hooks():
|
||||||
|
"""register() wires the four hooks."""
|
||||||
|
ctx = _Ctx()
|
||||||
|
hp.register(ctx)
|
||||||
|
names = {name for name, _ in ctx.hooks}
|
||||||
|
assert names == {"pre_llm_call", "post_api_request",
|
||||||
|
"post_auxiliary_call", "post_llm_call"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_plugin_usage_bucket_folds_cache(tmp_path):
|
||||||
|
"""CanonicalUsage disjoint buckets fold into inclusive ones."""
|
||||||
|
bucket = hp._usage_bucket({
|
||||||
|
"input_tokens": 50000, "cache_read_tokens": 400000,
|
||||||
|
"cache_write_tokens": 3000, "output_tokens": 9000,
|
||||||
|
})
|
||||||
|
assert bucket == {"input_tokens": 453000,
|
||||||
|
"cached_input_tokens": 400000,
|
||||||
|
"output_tokens": 9000}
|
||||||
|
|
||||||
|
|
||||||
|
def test_plugin_usage_bucket_none():
|
||||||
|
"""Missing/None usage is ignored."""
|
||||||
|
assert hp._usage_bucket(None) is None
|
||||||
|
assert hp._usage_bucket({}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_plugin_records_and_prints(tmp_path, monkeypatch, capsys):
|
||||||
|
"""A full simulated turn: record calls, then print the line."""
|
||||||
|
home = make_hermes_home(
|
||||||
|
tmp_path,
|
||||||
|
sessions=[("s1", "t", "zai-glm-5-3", 1.0)],
|
||||||
|
usage=[("s1", "zai-glm-5-3", 3, 100000, 20000, 900000, 5000)],
|
||||||
|
)
|
||||||
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||||
|
monkeypatch.setattr(hp, "_turn_usage", {})
|
||||||
|
monkeypatch.setattr(hp, "_turn_labels", {})
|
||||||
|
monkeypatch.setattr(hp.os, "environ",
|
||||||
|
{**os.environ, "HERMES_HOME": str(home)})
|
||||||
|
hp._on_pre_llm_call(turn_id="t1", user_message=" hello world ",
|
||||||
|
session_id="s1")
|
||||||
|
hp._on_post_api_request(
|
||||||
|
session_id="s1", turn_id="t1", model="zai-glm-5-3",
|
||||||
|
provider="zai", api_request_id="r1",
|
||||||
|
usage={"input_tokens": 50000, "cache_read_tokens": 400000,
|
||||||
|
"cache_write_tokens": 3000, "output_tokens": 9000})
|
||||||
|
hp._on_post_auxiliary_call(
|
||||||
|
session_id="s1", turn_id="t1", model="zai-glm-5-3",
|
||||||
|
provider="zai", aux_task="title_generation",
|
||||||
|
usage={"input_tokens": 500, "cache_read_tokens": 0,
|
||||||
|
"cache_write_tokens": 0, "output_tokens": 100})
|
||||||
|
hp._LEDGER_QUEUE.join()
|
||||||
|
hp._on_post_llm_call(session_id="s1", turn_id="t1",
|
||||||
|
model="zai-glm-5-3", platform="cli")
|
||||||
|
out = capsys.readouterr().out
|
||||||
|
assert "-- zai:zai-glm-5-3 | session:$0.44 | turn:$0.21" in out
|
||||||
|
# The ledger was written with the label from pre_llm_call
|
||||||
|
ledger = (home / "plugin-data" / "agent-cost-estimator" /
|
||||||
|
"usage" / "s1.jsonl").read_text()
|
||||||
|
assert '"label":"hello world"' in ledger
|
||||||
|
assert ledger.count("\n") == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_plugin_quiet_for_subagent(tmp_path, monkeypatch, capsys):
|
||||||
|
"""Subagent turns record but never print."""
|
||||||
|
home = make_hermes_home(
|
||||||
|
tmp_path,
|
||||||
|
sessions=[("s1", "t", "zai-glm-5-3", 1.0)],
|
||||||
|
usage=[("s1", "zai-glm-5-3", 1, 1000, 100, 0, 0)],
|
||||||
|
)
|
||||||
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||||
|
monkeypatch.setattr(hp, "_turn_usage", {})
|
||||||
|
hp._on_post_api_request(
|
||||||
|
session_id="s1", turn_id="t1", model="zai-glm-5-3",
|
||||||
|
provider="zai", usage={"input_tokens": 10,
|
||||||
|
"cache_read_tokens": 0,
|
||||||
|
"cache_write_tokens": 0,
|
||||||
|
"output_tokens": 5})
|
||||||
|
hp._on_post_llm_call(session_id="s1", turn_id="t1",
|
||||||
|
model="zai-glm-5-3", platform="subagent")
|
||||||
|
assert capsys.readouterr().out == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_plugin_quiet_for_single_query(tmp_path, monkeypatch, capsys):
|
||||||
|
"""hermes chat -q (single-query env) never prints."""
|
||||||
|
home = make_hermes_home(
|
||||||
|
tmp_path,
|
||||||
|
sessions=[("s1", "t", "zai-glm-5-3", 1.0)],
|
||||||
|
usage=[("s1", "zai-glm-5-3", 1, 1000, 100, 0, 0)],
|
||||||
|
)
|
||||||
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||||
|
monkeypatch.setenv("HERMES_SINGLE_QUERY_SESSION", "1")
|
||||||
|
monkeypatch.setattr(hp, "_turn_usage", {})
|
||||||
|
hp._on_post_llm_call(session_id="s1", turn_id="t1",
|
||||||
|
model="zai-glm-5-3", platform="cli")
|
||||||
|
assert capsys.readouterr().out == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_plugin_pricing_local_file(tmp_path, monkeypatch):
|
||||||
|
"""A plugin-local pricing.json is honored without the package."""
|
||||||
|
payload = {"providers": {"myp": {"currency": "USD", "models": {
|
||||||
|
"model-x": {"input_price": 1.0, "cached_input_price": 0.1,
|
||||||
|
"output_price": 5.0}}}}}
|
||||||
|
real_load = hp._load_local_pricing
|
||||||
|
|
||||||
|
def fake_load(path, _payload=payload):
|
||||||
|
if path.name == "pricing.json" and path.parent == pathlib.Path(
|
||||||
|
hp.__file__).parent:
|
||||||
|
return _payload
|
||||||
|
return real_load(path)
|
||||||
|
monkeypatch.setattr(hp, "_load_local_pricing", fake_load)
|
||||||
|
rates, key, is_default = hp._pricing("model-x")
|
||||||
|
assert key == "myp:model-x"
|
||||||
|
assert rates["input_price"] == 1.0
|
||||||
|
assert is_default is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_plugin_pricing_fallback():
|
||||||
|
"""No local file, no package import: hard-coded defaults."""
|
||||||
|
rates, key, is_default = hp._pricing("unknown-model-xyz")
|
||||||
|
assert is_default is True
|
||||||
|
assert rates["input_price"] == 1.5
|
||||||
|
|
||||||
|
|
||||||
|
# --- hermes installer ---
|
||||||
|
|
||||||
|
from agent_cost_estimator.integration.hermes import install as hinstall
|
||||||
|
|
||||||
|
|
||||||
|
def test_installer_dry_run(tmp_path, capsys):
|
||||||
|
"""--dry-run writes nothing."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[("s1", "t", "m", 1.0)])
|
||||||
|
code = hinstall.main(["--hermes-home", str(home), "--dry-run"])
|
||||||
|
assert code == 0
|
||||||
|
assert not (home / "plugins" / "agent-cost-estimator").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_installer_requires_hermes_home(tmp_path, capsys):
|
||||||
|
"""A directory without state.db/config.yaml is refused."""
|
||||||
|
empty = tmp_path / "empty"
|
||||||
|
empty.mkdir()
|
||||||
|
code = hinstall.main(["--hermes-home", str(empty)])
|
||||||
|
assert code == 1
|
||||||
|
assert "does not look like a Hermes home" in capsys.readouterr().err
|
||||||
|
|
||||||
|
|
||||||
|
def test_installer_copies_plugin(tmp_path, monkeypatch, capsys):
|
||||||
|
"""The install copies plugin files + pricing.json."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[("s1", "t", "m", 1.0)])
|
||||||
|
# hermes CLI not on PATH in tests: expect the manual-enable note
|
||||||
|
def fake_run(*args, **kwargs):
|
||||||
|
raise FileNotFoundError("no hermes")
|
||||||
|
monkeypatch.setattr(hinstall.subprocess, "run", fake_run)
|
||||||
|
code = hinstall.main(["--hermes-home", str(home)])
|
||||||
|
assert code == 0
|
||||||
|
dest = home / "plugins" / "agent-cost-estimator"
|
||||||
|
assert (dest / "__init__.py").is_file()
|
||||||
|
assert (dest / "plugin.yaml").is_file()
|
||||||
|
assert (dest / "pricing.json").is_file()
|
||||||
|
pricing = json.loads((dest / "pricing.json").read_text())
|
||||||
|
assert "providers" in pricing
|
||||||
|
assert "enable" in capsys.readouterr().out
|
||||||
|
|
||||||
|
|
||||||
|
def test_installer_uninstall(tmp_path, capsys):
|
||||||
|
"""--uninstall removes the plugin directory."""
|
||||||
|
home = make_hermes_home(tmp_path, sessions=[("s1", "t", "m", 1.0)])
|
||||||
|
dest = home / "plugins" / "agent-cost-estimator"
|
||||||
|
dest.mkdir(parents=True)
|
||||||
|
(dest / "__init__.py").write_text("# plugin")
|
||||||
|
code = hinstall.main(["--hermes-home", str(home), "--uninstall"])
|
||||||
|
assert code == 0
|
||||||
|
assert not dest.exists()
|
||||||
|
|||||||
Reference in New Issue
Block a user