diff --git a/.vibe/ai-usage-cost.md b/.vibe/ai-usage-cost.md index 9375d46..08edf7c 100644 --- a/.vibe/ai-usage-cost.md +++ b/.vibe/ai-usage-cost.md @@ -147,6 +147,23 @@ And add the cost information to this file for the current session. display form; GUI session rows and the refresh lookup resolver keep in sync. AGENTS.md documents the fields. MINOR bump to 0.5.0. +#### (pending) - feat: credit-pool display (mistral-api vs mistral-vibe) +- **Date**: 2026-11-10 +- **Model**: mistral-vibe:zai-glm-5-3 +- **Session Cost**: $9.41 (running total at commit time) +- **Turn Cost**: $1.10 +- **Total for commit**: $9.41 +- **Notes**: Credit pools differentiate billing pools that share a + pricing table: harness_credit_pools (vibe -> mistral-vibe) for + harnesses that do not log the billing API (Vibe), billing_url_pools + (api.mistral.ai -> mistral-api) driven by Hermes's logged + billing_base_url, which wins over the harness mapping. Pool + overrides the provider part of the display key; SessionMeta gains + base_url from the Hermes adapter; CLI --json gains credit_pool; + GUI session strings and the refresh resolver handle pool + prefixes. Verified live: mistral-vibe:zai-glm-5-3 (Vibe) and + mistral-api:mistral-large-4 (Hermes). MINOR bump to 0.6.0. + --- ## Instructions for Contributors diff --git a/AGENTS.md b/AGENTS.md index ca7f6de..b93c815 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -272,7 +272,18 @@ How to update prices in the future: prefix but another provider actually bills the model. The CLI's `--json` reports the display key as `pricing.model_key` and the raw pricing key as `pricing.model_key_raw`. -6. **Prefer editing `pricing.json` directly** so `note`/`source`/`date` stay +6. **Credit pools (optional)**: `harness_credit_pools` maps a harness to the + credit pool its sessions bill (`"vibe": "mistral-vibe"`, + `"hermes": "mistral-api"`) - use this when the harness logs do not record + the billing API (Vibe logs only the model name). `billing_url_pools` maps + a substring of the logged base URL to a pool + (`"api.mistral.ai": "mistral-api"`) - Hermes records + `session_model_usage.billing_base_url`, so its pool comes from the data + and wins over the harness mapping. The pool overrides the provider part of + the display key (`mistral-api:mistral-large-4` vs + `mistral-vibe:mistral-large-4`) and appears as `credit_pool` in the CLI's + `--json`; the rates themselves stay from the matched pricing entry. +7. **Prefer editing `pricing.json` directly** so `note`/`source`/`date` stay consistent, or use the CLI for rates-only updates: ```bash # Update pricing for a model (writes to the first provider's models) @@ -281,20 +292,15 @@ How to update prices in the future: # With custom pricing file agent-cost-estimator --pricing custom.json --model my-model --input-price 2.0 --update-pricing ``` -6. **Verify** against a real session before committing: +8. **Verify** against a real session before committing: ```bash python -m agent_cost_estimator -d # check "prices" and "TOTAL" lines python -m pytest tests/ -v # all tests must pass ``` -7. **Verify** against a real session before committing: - ```bash - python -m agent_cost_estimator -d # check "prices" and "TOTAL" lines - python -m pytest tests/ -v # all tests must pass - ``` -8. **Commit**: bump the version in `pyproject.toml` (PATCH for price/data - updates, MINOR for new features like display aliases), record the session - cost in `.vibe/ai-usage-cost.md`, and use `chore:` or `bump:` in the commit - message for pricing-only changes. +9. **Commit**: bump the version in `pyproject.toml` (PATCH for price/data + updates, MINOR for new features like display aliases or credit pools), + record the session cost in `.vibe/ai-usage-cost.md`, and use `chore:` or + `bump:` in the commit message for pricing-only changes. ### Testing Hooks diff --git a/agent_cost_estimator/cli.py b/agent_cost_estimator/cli.py index 1962065..cbf3a7d 100644 --- a/agent_cost_estimator/cli.py +++ b/agent_cost_estimator/cli.py @@ -14,7 +14,9 @@ from .pricing import ( DEFAULT_PRICING_PATH, RATES_KEYS, display_key_for, + get_harness_pool, get_model_prices, + get_url_pool, load_pricing, update_pricing, ) @@ -774,10 +776,13 @@ def main(argv=None): "output": usage.output_tokens, } cost = compute_cost(tokens, rates) - # Display form of the pricing key (provider_alias/model_alias - # applied, e.g. "zai:zai-glm-5-3" -> "mistral:zai-glm-5-3"); - # the raw key still drives --update-pricing above. - display_key = display_key_for(pricing, key) + # Display form of the pricing key: provider_alias/model_alias + # applied, then the credit pool (e.g. mistral-api vs mistral-vibe) + # overriding the provider part; the raw key still drives + # --update-pricing above. + pool = get_url_pool(pricing, meta.base_url) or get_harness_pool( + pricing, harness.name) + display_key = display_key_for(pricing, key, pool=pool) last_turn = next( (t for t in reversed(usage.turns) if t.calls), None) turn_cost = compute_cost( @@ -799,6 +804,7 @@ def main(argv=None): "title": meta.title, "model": meta.model_name, "provider": provider_model[0], + "credit_pool": pool, "provider_model": display_key, "calls": usage.calls, "input_tokens": usage.input_tokens, @@ -857,7 +863,9 @@ def main(argv=None): label = f" ({meta.title})" if meta.title else "" print(f"harness: {harness.name}") print(f"session: {session_dir.name}{label}") - print(f"model: {display_key}") # display key: provider_alias/model_alias applied + print(f"model: {display_key}") # display key: pool + aliases applied + if pool: + print(f"credit pool: {pool}") if is_default: print(" **model price not available**") print(f"model calls: {usage.calls} " diff --git a/agent_cost_estimator/gui.py b/agent_cost_estimator/gui.py index 48190a6..91a3ec8 100644 --- a/agent_cost_estimator/gui.py +++ b/agent_cost_estimator/gui.py @@ -228,6 +228,13 @@ def get_model_rates_for_display(pricing_data: dict[str, Any], return pricing_module.get_model_prices( pricing_data, model_name=model_name, provider=provider ) + # Not a raw or aliased pricing key: a credit-pool display string + # ("mistral-api:mistral-large-4") names the billing pool, not a + # provider section; look the model up across all providers. + if model_str and ":" in model_str and model_str.split(":", 1)[0] in ( + pricing_module.display_pools(pricing_data)): + return pricing_module.get_model_prices( + pricing_data, model_name=model_str.split(":", 1)[1]) # Bare model name (no provider qualifier): search all providers return pricing_module.get_model_prices(pricing_data, model_name=model_str) @@ -439,8 +446,10 @@ class SessionDiscovery: model_name=meta.model_name, alias=meta.alias ) + pool = (pricing_module.get_url_pool(self.pricing_data, meta.base_url) + or pricing_module.get_harness_pool(self.pricing_data, harness_name)) display_key = pricing_module.display_key_for( - self.pricing_data, matched_key) + self.pricing_data, matched_key, pool=pool) if display_key: model_str = display_key @@ -613,10 +622,14 @@ class SessionDiscovery: # Format model string model_str = model - + # If we have provider info from matched_key, format it + # (Hermes logs the billing base URL; use its pool) + base_url = getattr(adapter.read_meta(session_dir), "base_url", None) + pool = (pricing_module.get_url_pool(self.pricing_data, base_url) + or pricing_module.get_harness_pool(self.pricing_data, "hermes")) display_key = pricing_module.display_key_for( - self.pricing_data, matched_key) + self.pricing_data, matched_key, pool=pool) if display_key: model_str = display_key diff --git a/agent_cost_estimator/harnesses/base.py b/agent_cost_estimator/harnesses/base.py index ad96eb2..6afdb40 100644 --- a/agent_cost_estimator/harnesses/base.py +++ b/agent_cost_estimator/harnesses/base.py @@ -19,11 +19,17 @@ import pathlib @dataclasses.dataclass class SessionMeta: - """What a session is: display fields plus pricing keys.""" + """What a session is: display fields plus pricing keys. + + base_url is the API base URL the harness logs as billing this + session, when the harness records one (Hermes does, Vibe does + not). It drives the credit-pool display; None means unknown. + """ title: str | None = None model_name: str | None = None alias: str | None = None + base_url: str | None = None @dataclasses.dataclass diff --git a/agent_cost_estimator/harnesses/hermes.py b/agent_cost_estimator/harnesses/hermes.py index aa93d07..82e29b1 100644 --- a/agent_cost_estimator/harnesses/hermes.py +++ b/agent_cost_estimator/harnesses/hermes.py @@ -209,6 +209,27 @@ def _ledger_path(session_dir) -> pathlib.Path: / f"{session_dir.session_id}.jsonl") +def read_billing_base_url(session_dir): + """Return the API base URL Hermes logs as billing this session. + + session_model_usage.billing_base_url (e.g. + "https://api.mistral.ai/v1"), from any row of the session. None + when Hermes did not record one (older DB schemas, custom rows). + """ + try: + with _connect(session_dir.db_path) as con: + row = con.execute( + "SELECT billing_base_url FROM session_model_usage " + "WHERE session_id = ? AND billing_base_url IS NOT NULL " + "AND billing_base_url != '' LIMIT 1", + (session_dir.session_id,), + ).fetchone() + return row[0] if row else None + except sqlite3.OperationalError: + # Schema without the column (older Hermes versions) + return None + + def _fetch_one(session_dir, query, params=()): db_path = session_dir.db_path if not db_path.is_file(): @@ -255,7 +276,8 @@ class HermesHarness(HarnessAdapter): def read_meta(self, session_dir): title, model_name, alias = read_meta(session_dir) return SessionMeta(title=title, model_name=model_name, - alias=alias) + alias=alias, + base_url=read_billing_base_url(session_dir)) def collect_usage(self, session_dir): ledger_totals, ledger_turns = parse_ledger(session_dir) diff --git a/agent_cost_estimator/pricing.json b/agent_cost_estimator/pricing.json index 636180c..ee11b78 100644 --- a/agent_cost_estimator/pricing.json +++ b/agent_cost_estimator/pricing.json @@ -1,4 +1,11 @@ { + "harness_credit_pools": { + "vibe": "mistral-vibe", + "hermes": "mistral-api" + }, + "billing_url_pools": { + "api.mistral.ai": "mistral-api" + }, "providers": { "default": { "currency": "USD", diff --git a/agent_cost_estimator/pricing.py b/agent_cost_estimator/pricing.py index 82d3d61..84778a8 100644 --- a/agent_cost_estimator/pricing.py +++ b/agent_cost_estimator/pricing.py @@ -227,7 +227,46 @@ def get_model_prices(pricing, model_name=None, alias=None, provider=None): ) -def display_key_for(pricing: dict, matched_key: str | None) -> str | None: +def get_harness_pool(pricing: dict, harness_name: str | None) -> str | None: + """Harness -> credit pool label ("vibe" -> "mistral-vibe"). + + Optional table "harness_credit_pools" in the pricing file, for + harnesses whose logs do not record the billing API (Vibe logs + only the model name). None when unmapped. + """ + if not harness_name: + return None + mapping = pricing.get("harness_credit_pools") or {} + return mapping.get(harness_name) + + +def get_url_pool(pricing: dict, base_url: str | None) -> str | None: + """Billing API base URL -> credit pool label. + + Optional table "billing_url_pools" in the pricing file; rules + are substrings matched against the base URL the harness logged + ("api.mistral.ai" -> "mistral-api"). None when no rule matches. + """ + if not base_url: + return None + rules = pricing.get("billing_url_pools") or {} + for needle, pool in rules.items(): + if needle in base_url: + return pool + return None + + +def display_pools(pricing: dict) -> set: + """Every pool label that may appear as the provider part of a + display key (values of harness_credit_pools and + billing_url_pools).""" + labels = set((pricing.get("harness_credit_pools") or {}).values()) + labels.update((pricing.get("billing_url_pools") or {}).values()) + return labels + + +def display_key_for(pricing: dict, matched_key: str | None, + pool: str | None = None) -> str | None: """Return the display form of a matched key. Display aliases are optional pricing fields: @@ -235,18 +274,21 @@ def display_key_for(pricing: dict, matched_key: str | None) -> str | None: through Mistral's marketplace, sets provider_alias "mistral") - model-level "model_alias" - They change only how the cost tracker shows the model - ("mistral:zai-glm-5-3"), never the lookup. matched_key itself - stays the raw pricing key ("zai:zai-glm-5-3"). + pool overrides the provider part entirely (the credit pool the + session bills to, e.g. "mistral-api" vs "mistral-vibe"); it wins + over the provider_alias. Aliases and pool change only how the + cost tracker shows the model ("mistral-vibe:zai-glm-5-3"), never + the lookup; matched_key stays the raw pricing key ("zai:zai-glm-5-3"). """ if not matched_key or ":" not in matched_key: return matched_key provider, model = matched_key.split(":", 1) provider_data = (pricing.get("providers") or {}).get(provider) or {} provider_alias = provider_data.get("provider_alias") or provider - model_data = (provider_data.get("models") or {}).get(model) or {} - model_alias = model_data.get("model_alias") or model - return f"{provider_alias}:{model_alias}" + model_entry = (provider_data.get("models") or {}).get(model) or {} + model_alias = model_entry.get("model_alias") or model + shown_provider = pool or provider_alias + return f"{shown_provider}:{model_alias}" def resolve_display_key(pricing: dict, display_key: str | None) -> str | None: diff --git a/pyproject.toml b/pyproject.toml index 48b0ae9..fe16c8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "agent-cost-estimator" -version = "0.5.0" +version = "0.6.0" description = "Report token usage and estimated USD cost of AI agent sessions from local harness logs" readme = "README.md" requires-python = ">=3.9" diff --git a/tests/test_agent_cost_estimator.py b/tests/test_agent_cost_estimator.py index a60ec09..4209806 100755 --- a/tests/test_agent_cost_estimator.py +++ b/tests/test_agent_cost_estimator.py @@ -149,6 +149,41 @@ def test_display_key_for_applies_aliases(): assert pricing.display_key_for(data, None) is None +def test_credit_pool_helpers(): + """harness_credit_pools and billing_url_pools drive the pool.""" + data = pricing.load_pricing() + assert pricing.get_harness_pool(data, "vibe") == "mistral-vibe" + assert pricing.get_harness_pool(data, "hermes") == "mistral-api" + assert pricing.get_harness_pool(data, None) is None + assert pricing.get_harness_pool(data, "unknown") is None + assert pricing.get_url_pool( + data, "https://api.mistral.ai/v1") == "mistral-api" + assert pricing.get_url_pool(data, None) is None + assert pricing.get_url_pool(data, "https://example.com") is None + assert pricing.display_pools(data) == {"mistral-api", "mistral-vibe"} + + +def test_display_key_for_pool_overrides_provider(): + """A credit pool overrides the provider part of the display key, + and model_alias still applies.""" + data = pricing.load_pricing() + assert pricing.display_key_for( + data, "zai:zai-glm-5-3", pool="mistral-vibe") == \ + "mistral-vibe:zai-glm-5-3" + assert pricing.display_key_for( + data, "mistral:mistral-large-4", pool="mistral-api") == \ + "mistral-api:mistral-large-4" + data2 = {"providers": { + "zai": {"currency": "USD", + "models": { + "zai-glm-5-3": { + "input_price": 1.4, "cached_input_price": 0.14, + "output_price": 4.4, "model_alias": "glm-5.3"}}}}} + assert pricing.display_key_for( + data2, "zai:zai-glm-5-3", pool="mistral-vibe") == \ + "mistral-vibe:glm-5.3" + + def test_display_key_for_model_alias(): """model_alias overrides the model part of the display key.""" data = {"providers": { @@ -407,8 +442,8 @@ def test_vibe_default_home(): # --- cli --- def test_cli_shows_provider_alias_display_key(tmp_path, capsys): - """The default line displays the provider_alias form - (mistral:zai-glm-5-3) while lookup stays on the zai rates.""" + """The default line displays the pool form (mistral-vibe:) + while lookup stays on the zai rates.""" session = tmp_path / "logs" / "session" / "unified" / "s1" write_journal(session, [ user_message("hi"), @@ -419,7 +454,7 @@ def test_cli_shows_provider_alias_display_key(tmp_path, capsys): str(tmp_path)]) assert code == 0 out = capsys.readouterr().out.strip() - assert out.startswith("mistral:zai-glm-5-3 | session:$") + assert out.startswith("mistral-vibe:zai-glm-5-3 | session:$") # priced at GLM rates ($1.40/M), not the mistral fallback ($1.50/M) session_cost = (500_000 * 1.40 + 500_000 * 0.14) / 1e6 assert f"session:${session_cost:.2f}" in out @@ -439,9 +474,10 @@ def test_cli_json_reports_display_and_raw_key(tmp_path, capsys): str(tmp_path), "--json"]) assert code == 0 data = json.loads(capsys.readouterr().out) - assert data["provider"] == "mistral" - assert data["provider_model"] == "mistral:zai-glm-5-3" - assert data["pricing"]["model_key"] == "mistral:zai-glm-5-3" + assert data["credit_pool"] == "mistral-vibe" + assert data["provider"] == "mistral-vibe" + assert data["provider_model"] == "mistral-vibe:zai-glm-5-3" + assert data["pricing"]["model_key"] == "mistral-vibe:zai-glm-5-3" assert data["pricing"]["model_key_raw"] == "zai:zai-glm-5-3" @@ -519,7 +555,7 @@ def test_cli_default_compact_line(tmp_path, capsys): # Output format is now: provider:model | session:$X | turn:$Y # followed by: -- I:X O:Y C:Z # Model 'm1' is not in pricing, so it falls back to default with warning - expected = (f"default:default | session:${session_cost:.2f} | " + expected = (f"mistral-vibe:default | session:${session_cost:.2f} | " f"turn:${turn_cost:.2f}\n **model price not available**\n" f"-- I:2.00M O:100.00k C:500.00k") assert out == expected @@ -635,11 +671,13 @@ import sqlite3 from agent_cost_estimator.harnesses import hermes as hh -def make_hermes_home(tmp_path, sessions=(), usage=()): +def make_hermes_home(tmp_path, sessions=(), usage=(), base_urls=None): """Create a fake Hermes home: state.db + optional rows. sessions: (id, title, model, last_activity_at) usage: (session_id, model, calls, input, output, read, write) + base_urls: {session_id: billing_base_url} - adds the column and + sets it per session (Hermes logs the billing API there) """ home = tmp_path / "hermes-home" home.mkdir() @@ -658,6 +696,13 @@ def make_hermes_home(tmp_path, sessions=(), usage=()): for row in usage: con.execute("INSERT INTO session_model_usage VALUES " "(?, ?, '', ?, ?, ?, ?, ?, 0)", row) + if base_urls: + con.execute("ALTER TABLE session_model_usage " + "ADD COLUMN billing_base_url TEXT") + for session_id, url in base_urls.items(): + con.execute( + "UPDATE session_model_usage SET billing_base_url = ? " + "WHERE session_id = ?", (url, session_id)) con.commit() con.close() return home @@ -818,6 +863,69 @@ def test_hermes_cli_json(tmp_path, capsys): assert data["usage_source"] == "state.db" +def test_hermes_read_billing_base_url(tmp_path): + """The Hermes adapter exposes the logged billing API base URL, + tolerated when the schema lacks the column.""" + home = make_hermes_home( + tmp_path, + sessions=[("s1", "t", "mistral-large-4", 1.0), + ("s2", "t", "zai-glm-5-3", 2.0)], + usage=[("s1", "mistral-large-4", 1, 100, 0, 10, 0), + ("s2", "zai-glm-5-3", 1, 100, 0, 10, 0)], + base_urls={"s1": "https://api.mistral.ai/v1"}, + ) + harness = hh.HermesHarness() + meta1 = harness.read_meta(harness.resolve_session_dir("s1", home)) + assert meta1.base_url == "https://api.mistral.ai/v1" + # no URL logged for s2 (custom rows without billing info) + meta2 = harness.read_meta(harness.resolve_session_dir("s2", home)) + assert meta2.base_url is None + # old schema without the column still resolves meta + (tmp_path / "old").mkdir() + old = make_hermes_home( + tmp_path / "old", + sessions=[("s1", "t", "mistral-large-4", 1.0)], + usage=[("s1", "mistral-large-4", 1, 100, 0, 10, 0)], + ) + meta_old = harness.read_meta(harness.resolve_session_dir("s1", old)) + assert meta_old.base_url is None + + +def test_hermes_cli_pool_from_billing_url(tmp_path, capsys): + """A Hermes session billed against the Mistral API shows the + mistral-api credit pool, even for a model named mistral-vibe-*.""" + home = make_hermes_home( + tmp_path, + sessions=[("s1", "t", "mistral-vibe-cli-latest", 1.0)], + usage=[("s1", "mistral-vibe-cli-latest", 2, 1000, 500, 200, 0)], + base_urls={"s1": "https://api.mistral.ai/v1"}, + ) + code = cli.main(["--harness", "hermes", "--home", str(home), + "--json"]) + assert code == 0 + data = json.loads(capsys.readouterr().out) + assert data["credit_pool"] == "mistral-api" + assert data["provider"] == "mistral-api" + assert data["provider_model"] == "mistral-api:mistral-vibe-cli-latest" + assert data["pricing"]["model_key_raw"] == "mistral:mistral-vibe-cli-latest" + + +def test_hermes_cli_pool_falls_back_to_harness_mapping(tmp_path, capsys): + """No logged billing URL: the harness_credit_pools mapping + (hermes -> mistral-api) names the pool.""" + home = make_hermes_home( + tmp_path, + sessions=[("s1", "t", "mistral-large-4", 1.0)], + usage=[("s1", "mistral-large-4", 1, 1000, 500, 200, 0)], + ) + code = cli.main(["--harness", "hermes", "--home", str(home), + "--json"]) + assert code == 0 + data = json.loads(capsys.readouterr().out) + assert data["credit_pool"] == "mistral-api" + assert data["provider_model"] == "mistral-api:mistral-large-4" + + # --- hermes plugin (integration) --- from agent_cost_estimator.integration.hermes import plugin as hp diff --git a/tests/test_gui.py b/tests/test_gui.py index de0a2c7..ceb47f5 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -103,6 +103,21 @@ class TestGetModelRatesForDisplay: assert rates["input_price"] == 1.4 assert rates["output_price"] == 4.4 + def test_pool_display_resolves_to_real_rates(self): + """Pool-prefixed display keys ("mistral-vibe:zai-glm-5-3", + "mistral-api:mistral-large-4") resolve to the model's own + rates; the pool names a billing pool, not a provider.""" + pricing = load_pricing() + rates, key, _, _ = get_model_rates_for_display( + pricing, "mistral-vibe:zai-glm-5-3") + assert key == "zai:zai-glm-5-3" + assert rates["output_price"] == 4.4 + rates, key, _, _ = get_model_rates_for_display( + pricing, "mistral-api:mistral-large-4") + assert key == "mistral:mistral-large-4" + assert rates["input_price"] == 0.7 + assert rates["output_price"] == 2.1 + def test_bare_model_falls_back_to_default(self): """An unqualified unknown model still gets the fallback rates.""" pricing = load_pricing()