Initial commit: agent-cost-estimator 0.1.0
- Python CLI tool to report token usage and estimated USD cost - Supports Mistral Vibe harness with pluggable adapter interface - Includes SKILL.md in skill/ directory for LLM consumption via --skill flag - Full test suite (28 tests) - Typical Python project structure with .gitignore, .gitattributes, LICENSE, README.md - Build script for PyPI distribution Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
# Set default line ending to LF
|
||||
* text=auto eol=lf
|
||||
|
||||
# Files that should maintain their line endings
|
||||
*.md text eol=lf
|
||||
*.txt text eol=lf
|
||||
*.py text eol=lf
|
||||
*.toml text eol=lf
|
||||
*.json text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
|
||||
# Binary files
|
||||
*.png binary
|
||||
*.jpg binary
|
||||
*.jpeg binary
|
||||
*.gif binary
|
||||
*.pdf binary
|
||||
*.zip binary
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# PyInstaller
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Local overrides
|
||||
*.local
|
||||
|
||||
# Build artifacts
|
||||
*.whl
|
||||
*.tar.gz
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 nvaldez
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,3 @@
|
||||
include README.md
|
||||
include LICENSE
|
||||
recursive-include agent_cost_estimator/skill *.md
|
||||
@@ -0,0 +1,106 @@
|
||||
# agent-cost-estimator
|
||||
|
||||
Report token usage and estimated USD cost of an AI agent session,
|
||||
parsed from the harness's local logs. Ships with a
|
||||
**Mistral Vibe** adapter; other AI harnesses plug in through the
|
||||
adapter interface.
|
||||
|
||||
The result is a local **estimate** priced from `pricing.json`. The
|
||||
authoritative monthly usage is your provider's console (for
|
||||
Mistral: admin.mistral.ai/subscription).
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
pip install . # from this directory
|
||||
# or run in place:
|
||||
python -m agent_cost_estimator --help
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
agent-cost-estimator # session: $2.11, turn: $2.50
|
||||
agent-cost-estimator -d # the full report table
|
||||
agent-cost-estimator -s # session cost only, e.g. 2.1148
|
||||
agent-cost-estimator -t # latest turn cost only
|
||||
agent-cost-estimator -d --turns # table plus per-turn breakdown
|
||||
agent-cost-estimator --session-id 5732da2a # a specific session id or path
|
||||
agent-cost-estimator --json # machine-readable output
|
||||
agent-cost-estimator --list-harnesses # available adapters
|
||||
```
|
||||
|
||||
With no output flag the tool prints one line: what the whole
|
||||
session costs and what the latest turn cost. `-d/--details`
|
||||
prints the full table; `-s/--session` and `-t/--turn` print one
|
||||
bare number each, for scripting. The turn cost is the most
|
||||
recent turn in the journal window.
|
||||
|
||||
Prices come from the bundled `agent_cost_estimator/pricing.json`,
|
||||
keyed by model (with a `default` fallback). Bring your own table:
|
||||
|
||||
```bash
|
||||
agent-cost-estimator --pricing my-prices.json
|
||||
```
|
||||
|
||||
Or persist overrides into the pricing table (the built-in one, or
|
||||
the file passed with `--pricing`):
|
||||
|
||||
```bash
|
||||
agent-cost-estimator --model zai-glm-5-3 \
|
||||
--input-price 1.2 --cached-input-price 0.12 --output-price 6.0 \
|
||||
--update-pricing # add --dry-run to preview
|
||||
```
|
||||
|
||||
## pricing.json format
|
||||
|
||||
```json
|
||||
{
|
||||
"currency": "USD",
|
||||
"models": {
|
||||
"default": {"input_price": 1.5, "cached_input_price": 0.15, "output_price": 7.5},
|
||||
"zai-glm-5-3": {"input_price": 1.5, "cached_input_price": 0.15, "output_price": 7.5}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Prices are USD per million tokens. Lookup order: session's model
|
||||
name, then its alias, then `default`. On the command line,
|
||||
`--input-price`, `--cached-input-price`, and `--output-price`
|
||||
override the table for that run.
|
||||
|
||||
## Adding a harness
|
||||
|
||||
A harness adapter is a subclass of
|
||||
`agent_cost_estimator.harnesses.base.HarnessAdapter` implementing
|
||||
three read-only methods:
|
||||
|
||||
- `resolve_session_dir(session_arg, home)` - find the session
|
||||
directory (or the most recently active one when the argument is
|
||||
None)
|
||||
- `read_meta(session_dir)` - return a `SessionMeta` (title,
|
||||
model_name, alias)
|
||||
- `collect_usage(session_dir)` - return a `SessionUsage`
|
||||
(input/cached/output tokens, call count, per-turn list)
|
||||
|
||||
Set `name`, `home_env_var`, and `home_dir_name` so `--home` and
|
||||
the session lookup have a sane default root. Register the adapter
|
||||
either by adding it to `BUILTIN_HARNESSES` (in-tree) or by
|
||||
exposing it through the `agent_cost_estimator.harnesses`
|
||||
entry-point group from your own package:
|
||||
|
||||
```toml
|
||||
[project.entry-points."agent_cost_estimator.harnesses"]
|
||||
mytool = "my_package.adapters:MyToolHarness"
|
||||
```
|
||||
|
||||
Built-ins always win a name clash; a broken plugin is skipped with
|
||||
a warning.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
python -m pytest tests/
|
||||
```
|
||||
|
||||
Standard library only; no runtime dependencies.
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Estimate the USD cost of AI agent sessions from local logs.
|
||||
|
||||
agent-cost-estimator reads a session's token usage from an AI
|
||||
coding harness's on-disk logs (Vibe today, more harnesses via the
|
||||
plugin registry), prices it from a JSON pricing table, and prints
|
||||
the result.
|
||||
|
||||
Package layout:
|
||||
- core: harness-independent cost math
|
||||
- pricing: pricing table loading, lookup, updates
|
||||
- harnesses: adapter registry; one adapter per AI harness
|
||||
- cli: the agent-cost-estimator command
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .core import compute_cost
|
||||
from .harnesses import (
|
||||
HarnessAdapter,
|
||||
available_harnesses,
|
||||
get_harness,
|
||||
)
|
||||
from .pricing import (
|
||||
DEFAULT_PRICING_PATH,
|
||||
RATES_KEYS,
|
||||
get_model_prices,
|
||||
load_pricing,
|
||||
update_pricing,
|
||||
)
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_PRICING_PATH",
|
||||
"RATES_KEYS",
|
||||
"HarnessAdapter",
|
||||
"__version__",
|
||||
"available_harnesses",
|
||||
"compute_cost",
|
||||
"get_harness",
|
||||
"get_model_prices",
|
||||
"load_pricing",
|
||||
"update_pricing",
|
||||
]
|
||||
@@ -0,0 +1,8 @@
|
||||
"""Allow python -m agent_cost_estimator."""
|
||||
|
||||
import sys
|
||||
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Command-line interface for agent_cost_estimator."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
from .core import compute_cost
|
||||
from .harnesses import available_harnesses, get_harness
|
||||
from .pricing import (
|
||||
DEFAULT_PRICING_PATH,
|
||||
RATES_KEYS,
|
||||
get_model_prices,
|
||||
load_pricing,
|
||||
update_pricing,
|
||||
)
|
||||
|
||||
LOG = logging.getLogger("agent_cost_estimator.cli")
|
||||
|
||||
# SKILL.md is in the skill/ subdirectory of the package
|
||||
SKILL_PATH = pathlib.Path(__file__).parent / "skill" / "SKILL.md"
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="agent-cost-estimator",
|
||||
description="Report token usage and estimated USD cost of "
|
||||
"an AI agent session, from the harness's local logs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--harness", default="vibe",
|
||||
help="which AI harness to read (default: vibe; run "
|
||||
"--list-harnesses to see all)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list-harnesses", action="store_true",
|
||||
help="print the available harness names and exit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skill", action="store_true",
|
||||
help="print the SKILL.md usage instructions (for LLMs) "
|
||||
"and exit; no session is read",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--session-id", default=None,
|
||||
help="session id or session directory (default: most "
|
||||
"recently active session)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--home", default=None,
|
||||
help="harness data root, e.g. the Vibe home (default: "
|
||||
"harness-specific, e.g. $VIBE_HOME or ~/.vibe)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pricing", default=None,
|
||||
help="path to a pricing JSON file (default: the bundled "
|
||||
"pricing.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model", default=None,
|
||||
help="force the pricing lookup key for this run "
|
||||
"(default: the session's model name, then alias, "
|
||||
'then "default")',
|
||||
)
|
||||
parser.add_argument("--input-price", type=float, default=None,
|
||||
help="override: USD per million input tokens")
|
||||
parser.add_argument("--cached-input-price", type=float, default=None,
|
||||
help="override: USD per million cached input "
|
||||
"tokens")
|
||||
parser.add_argument("--output-price", type=float, default=None,
|
||||
help="override: USD per million output tokens")
|
||||
parser.add_argument(
|
||||
"-d", "--details", action="store_true",
|
||||
help="the full report table (cost lines, sources, notes)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s", "--session", action="store_true",
|
||||
help="print only the session cost, as a bare number",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t", "--turn", action="store_true",
|
||||
help="print only the latest turn's cost, as a bare number",
|
||||
)
|
||||
parser.add_argument("--turns", action="store_true",
|
||||
help="with --details: also print a per-turn "
|
||||
"breakdown")
|
||||
parser.add_argument("--json", action="store_true",
|
||||
help="print one JSON object instead of a table")
|
||||
parser.add_argument(
|
||||
"--update-pricing", action="store_true",
|
||||
help="persist the effective rates (file rates plus price "
|
||||
"overrides) for the model into the pricing file; "
|
||||
"requires at least one price override",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true",
|
||||
help="with --update-pricing: print what would change "
|
||||
"without writing",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def effective_rates(args, pricing, meta):
|
||||
"""Return (rates, key) after table lookup and CLI overrides."""
|
||||
rates, key = get_model_prices(pricing, args.model or meta.model_name,
|
||||
meta.alias)
|
||||
overrides = {
|
||||
"input_price": args.input_price,
|
||||
"cached_input_price": args.cached_input_price,
|
||||
"output_price": args.output_price,
|
||||
}
|
||||
rates.update({k: v for k, v in overrides.items() if v is not None})
|
||||
return rates, key
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
args = parse_args(argv)
|
||||
logging.basicConfig(
|
||||
level=logging.WARNING,
|
||||
stream=sys.stderr,
|
||||
format="%(levelname)s %(name)s %(message)s",
|
||||
)
|
||||
if args.skill:
|
||||
try:
|
||||
print(SKILL_PATH.read_text(encoding="utf-8"), end="")
|
||||
except OSError as exc:
|
||||
print(f"error: cannot read the skill doc: {exc}",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
if args.list_harnesses:
|
||||
for name in sorted(available_harnesses()):
|
||||
print(name)
|
||||
return 0
|
||||
if args.dry_run and not args.update_pricing:
|
||||
print("error: --dry-run needs --update-pricing",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
if args.update_pricing and not any(
|
||||
getattr(args, flag) is not None
|
||||
for flag in ("input_price", "cached_input_price", "output_price")
|
||||
):
|
||||
print("error: --update-pricing needs at least one of "
|
||||
"--input-price, --cached-input-price, --output-price",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
pricing_path = pathlib.Path(args.pricing) if args.pricing else None
|
||||
try:
|
||||
harness = get_harness(args.harness)
|
||||
home = (pathlib.Path(args.home).expanduser()
|
||||
if args.home else harness.default_home())
|
||||
session_dir = harness.resolve_session_dir(args.session_id, home)
|
||||
meta = harness.read_meta(session_dir)
|
||||
usage = harness.collect_usage(session_dir)
|
||||
pricing = load_pricing(pricing_path)
|
||||
rates, key = effective_rates(args, pricing, meta)
|
||||
except (FileNotFoundError, TypeError, ValueError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if args.update_pricing:
|
||||
target = args.model or meta.model_name or meta.alias or "default"
|
||||
write_path, written = update_pricing(
|
||||
pricing_path or DEFAULT_PRICING_PATH,
|
||||
target,
|
||||
{k: rates[k] for k in RATES_KEYS},
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
state = "updated" if written else "would update"
|
||||
print(f"{state} {target} in {write_path}", file=sys.stderr)
|
||||
tokens = {
|
||||
"fresh": usage.fresh_input_tokens,
|
||||
"cached": usage.cached_input_tokens,
|
||||
"output": usage.output_tokens,
|
||||
}
|
||||
cost = compute_cost(tokens, rates)
|
||||
last_turn = next(
|
||||
(t for t in reversed(usage.turns) if t.calls), None)
|
||||
turn_cost = compute_cost(
|
||||
{"fresh": last_turn.fresh_input_tokens,
|
||||
"cached": last_turn.cached_input_tokens,
|
||||
"output": last_turn.output_tokens},
|
||||
rates,
|
||||
) if last_turn else {
|
||||
"input_usd": 0.0, "cached_usd": 0.0, "output_usd": 0.0,
|
||||
"total_usd": 0.0,
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps({
|
||||
"harness": harness.name,
|
||||
"session_id": session_dir.name,
|
||||
"title": meta.title,
|
||||
"model": meta.model_name,
|
||||
"calls": usage.calls,
|
||||
"input_tokens": usage.input_tokens,
|
||||
"cached_input_tokens": usage.cached_input_tokens,
|
||||
"fresh_input_tokens": usage.fresh_input_tokens,
|
||||
"output_tokens": usage.output_tokens,
|
||||
"pricing": {
|
||||
"file": str(pricing_path) if pricing_path
|
||||
else "bundled",
|
||||
"model_key": key,
|
||||
"rates_usd_per_million": rates,
|
||||
},
|
||||
"usage_source": usage.usage_source,
|
||||
"cost_usd": cost,
|
||||
"last_turn": {
|
||||
"label": last_turn.label if last_turn else None,
|
||||
"cost_usd": turn_cost,
|
||||
} if last_turn else None,
|
||||
"turns": [
|
||||
{"label": t.label, "calls": t.calls,
|
||||
"input_tokens": t.input_tokens,
|
||||
"cached_input_tokens": t.cached_input_tokens,
|
||||
"output_tokens": t.output_tokens,
|
||||
"fresh_input_tokens": t.fresh_input_tokens}
|
||||
for t in usage.turns if t.calls
|
||||
],
|
||||
}, indent=2))
|
||||
return 0
|
||||
if args.session and not args.turn:
|
||||
print(f"{cost['total_usd']:.4f}")
|
||||
return 0
|
||||
if args.turn and not args.session:
|
||||
print(f"{turn_cost['total_usd']:.4f}")
|
||||
return 0
|
||||
if not args.details:
|
||||
# default: the quick answer - what the session costs and
|
||||
# what the latest turn cost
|
||||
print(f"session: ${cost['total_usd']:.2f}, "
|
||||
f"turn: ${turn_cost['total_usd']:.2f}")
|
||||
return 0
|
||||
label = f" ({meta.title})" if meta.title else ""
|
||||
print(f"harness: {harness.name}")
|
||||
print(f"session: {session_dir.name}{label}")
|
||||
if meta.model_name or meta.alias:
|
||||
print(f"model: {meta.model_name or meta.alias}")
|
||||
print(f"model calls: {usage.calls} "
|
||||
f"(window on disk; pruned calls not counted)")
|
||||
print(f"usage source: {usage.usage_source} "
|
||||
f"(session-wide totals)")
|
||||
print(f"input tokens: {usage.input_tokens:,} "
|
||||
f"(cached {usage.cached_input_tokens:,}, "
|
||||
f"fresh {usage.fresh_input_tokens:,})")
|
||||
print(f"output tokens: {usage.output_tokens:,}")
|
||||
print(f"pricing: {key} from "
|
||||
f"{pricing_path if pricing_path else 'bundled pricing.json'}")
|
||||
print(f"prices (USD/M): input {rates['input_price']} "
|
||||
f"cached {rates['cached_input_price']} "
|
||||
f"output {rates['output_price']}")
|
||||
print(f"input cost: ${cost['input_usd']:.4f}")
|
||||
print(f"cached input cost: ${cost['cached_usd']:.4f}")
|
||||
print(f"output cost: ${cost['output_usd']:.4f}")
|
||||
print(f"TOTAL: ${cost['total_usd']:.4f}")
|
||||
print("note: local estimate at the given rates; authoritative")
|
||||
print(" usage is the provider's console")
|
||||
if args.turns:
|
||||
print()
|
||||
print("turns:")
|
||||
for turn in usage.turns:
|
||||
if not turn.calls:
|
||||
continue
|
||||
turn_cost = compute_cost(
|
||||
{"fresh": turn.fresh_input_tokens,
|
||||
"cached": turn.cached_input_tokens,
|
||||
"output": turn.output_tokens},
|
||||
rates)
|
||||
print(f" {turn.calls:>3} calls | "
|
||||
f"in {turn.input_tokens:>8,} "
|
||||
f"(cached {turn.cached_input_tokens:>8,}) | "
|
||||
f"out {turn.output_tokens:>7,} | "
|
||||
f"${turn_cost['total_usd']:.4f} | {turn.label!r}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Generic token-cost math for agent_cost_estimator.
|
||||
|
||||
Harness-independent: works on the token counts a harness adapter
|
||||
reports and a USD-per-million pricing table entry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def compute_cost(tokens, rates):
|
||||
"""Return the USD cost dict for the token totals.
|
||||
|
||||
tokens: dict with fresh, cached, and output token counts
|
||||
(keys: "fresh", "cached", "output").
|
||||
rates: dict with input_price, cached_input_price, and
|
||||
output_price in USD per million tokens.
|
||||
"""
|
||||
input_cost = tokens["fresh"] * rates["input_price"] / 1_000_000
|
||||
cached_cost = tokens["cached"] * rates["cached_input_price"] / 1_000_000
|
||||
output_cost = tokens["output"] * rates["output_price"] / 1_000_000
|
||||
return {
|
||||
"input_usd": round(input_cost, 4),
|
||||
"cached_usd": round(cached_cost, 4),
|
||||
"output_usd": round(output_cost, 4),
|
||||
"total_usd": round(input_cost + cached_cost + output_cost, 4),
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Harness registry.
|
||||
|
||||
Built-in adapters are listed in BUILTIN_HARNESSES. Third-party
|
||||
packages add adapters through the "agent_cost_estimator.harnesses"
|
||||
entry-point group, each entry pointing at a HarnessAdapter
|
||||
subclass:
|
||||
|
||||
[project.entry-points."agent_cost_estimator.harnesses"]
|
||||
mytool = "my_package.my_module:MyToolHarness"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.metadata
|
||||
import logging
|
||||
|
||||
from .base import HarnessAdapter, SessionMeta, SessionUsage, TurnUsage
|
||||
from .vibe import VibeHarness
|
||||
|
||||
LOG = logging.getLogger("agent_cost_estimator.harnesses")
|
||||
|
||||
ENTRY_POINT_GROUP = "agent_cost_estimator.harnesses"
|
||||
|
||||
BUILTIN_HARNESSES: dict[str, type[HarnessAdapter]] = {
|
||||
VibeHarness.name: VibeHarness,
|
||||
}
|
||||
|
||||
|
||||
def available_harnesses():
|
||||
"""Return the registry: name -> adapter class.
|
||||
|
||||
Built-ins first; entry-point plugins of the same name never
|
||||
override a built-in.
|
||||
"""
|
||||
registry = dict(BUILTIN_HARNESSES)
|
||||
try:
|
||||
eps = importlib.metadata.entry_points(group=ENTRY_POINT_GROUP)
|
||||
except TypeError: # Python 3.9: no group= filter
|
||||
eps = importlib.metadata.entry_points().get(ENTRY_POINT_GROUP, [])
|
||||
for ep in eps:
|
||||
if ep.name in registry:
|
||||
continue
|
||||
try:
|
||||
loaded = ep.load()
|
||||
except Exception as exc: # noqa: BLE001 - a broken plugin
|
||||
LOG.warning("skipping harness plugin %s: %s", ep.name, exc)
|
||||
continue
|
||||
if isinstance(loaded, type) and issubclass(loaded, HarnessAdapter):
|
||||
registry[ep.name] = loaded
|
||||
else:
|
||||
LOG.warning(
|
||||
"harness plugin %s is not a HarnessAdapter subclass",
|
||||
ep.name,
|
||||
)
|
||||
return registry
|
||||
|
||||
|
||||
def get_harness(name):
|
||||
"""Return an adapter instance for the harness name.
|
||||
|
||||
Raises ValueError when the name is unknown.
|
||||
"""
|
||||
registry = available_harnesses()
|
||||
if name not in registry:
|
||||
raise ValueError(
|
||||
f"unknown harness {name!r}; available: "
|
||||
f"{', '.join(sorted(registry))}"
|
||||
)
|
||||
return registry[name]()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BUILTIN_HARNESSES",
|
||||
"ENTRY_POINT_GROUP",
|
||||
"HarnessAdapter",
|
||||
"SessionMeta",
|
||||
"SessionUsage",
|
||||
"TurnUsage",
|
||||
"VibeHarness",
|
||||
"available_harnesses",
|
||||
"get_harness",
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Harness adapter interface for agent_cost_estimator.
|
||||
|
||||
A harness is one AI coding agent that keeps session logs on disk
|
||||
(Mistral Vibe, Claude Code, ...). Each harness is supported by one
|
||||
HarnessAdapter subclass that knows where its sessions live and how
|
||||
to read a session's token usage. Adapters are registered by name in
|
||||
the harness registry (agent_cost_estimator.harnesses) and, for
|
||||
third-party packages, through the "agent_cost_estimator.harnesses"
|
||||
entry-point group.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class SessionMeta:
|
||||
"""What a session is: display fields plus pricing keys."""
|
||||
|
||||
title: str | None = None
|
||||
model_name: str | None = None
|
||||
alias: str | None = None
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TurnUsage:
|
||||
"""Usage of one user turn, for breakdowns."""
|
||||
|
||||
label: str = ""
|
||||
calls: int = 0
|
||||
input_tokens: int = 0
|
||||
cached_input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
|
||||
@property
|
||||
def fresh_input_tokens(self) -> int:
|
||||
return self.input_tokens - self.cached_input_tokens
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class SessionUsage:
|
||||
"""Session-wide token usage, however the harness counts it.
|
||||
|
||||
usage_source names where the totals came from (harness
|
||||
specific, e.g. "projection-state" vs "journal" for Vibe), so
|
||||
the CLI can say how trustworthy the window is.
|
||||
"""
|
||||
|
||||
input_tokens: int = 0
|
||||
cached_input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
calls: int = 0
|
||||
usage_source: str = "unknown"
|
||||
turns: list[TurnUsage] = dataclasses.field(default_factory=list)
|
||||
|
||||
@property
|
||||
def fresh_input_tokens(self) -> int:
|
||||
return self.input_tokens - self.cached_input_tokens
|
||||
|
||||
|
||||
class HarnessAdapter(abc.ABC):
|
||||
"""Reads sessions of one AI harness from local logs.
|
||||
|
||||
Subclasses set `name`, `home_env_var`, and `home_dir_name`
|
||||
(the directory under $HOME when the env var is unset), then
|
||||
implement the three methods. All methods are read-only.
|
||||
"""
|
||||
|
||||
name: str = "abstract"
|
||||
home_env_var: str | None = None
|
||||
home_dir_name: str | None = None
|
||||
|
||||
def default_home(self) -> pathlib.Path:
|
||||
"""The harness's data root: $home_env_var or ~/.home_dir_name."""
|
||||
if self.home_env_var and os.environ.get(self.home_env_var):
|
||||
return pathlib.Path(os.environ[self.home_env_var])
|
||||
if self.home_dir_name:
|
||||
return pathlib.Path.home() / self.home_dir_name
|
||||
raise ValueError(
|
||||
f"harness {self.name!r} defines neither home_env_var "
|
||||
"nor home_dir_name; pass --home explicitly"
|
||||
)
|
||||
|
||||
@abc.abstractmethod
|
||||
def resolve_session_dir(self, session_arg, home):
|
||||
"""Return the directory of the session named by session_arg.
|
||||
|
||||
session_arg may be a session directory path, a session id,
|
||||
or None (pick the most recently active session). Raises
|
||||
FileNotFoundError with a message when nothing matches.
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def read_meta(self, session_dir):
|
||||
"""Return the SessionMeta for the session directory."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def collect_usage(self, session_dir):
|
||||
"""Return the SessionUsage for the session directory."""
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Vibe (Mistral, Unified Harness) adapter.
|
||||
|
||||
Sessions live under <home>/logs/session/unified/<uuid>/:
|
||||
- meta.json: title and config (model_name, active_model alias)
|
||||
- generations/*/projection-state.json: the newest holds the
|
||||
cumulative session tokenUsage counter, which survives journal
|
||||
rotation - the authoritative session total
|
||||
- journal/*.jsonl: per-call usage records (pruned over time, so
|
||||
call counts and per-turn breakdowns cover only the window that
|
||||
remains on disk)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
|
||||
from .base import HarnessAdapter, SessionMeta, SessionUsage, TurnUsage
|
||||
|
||||
LOG = logging.getLogger("agent_cost_estimator.harnesses.vibe")
|
||||
|
||||
|
||||
def resolve_session_dir(session_arg, home):
|
||||
"""Return the Vibe session directory for the argument.
|
||||
|
||||
Accepts a session directory path, a session id, or None (pick
|
||||
the most recently active session by journal file mtime).
|
||||
Raises FileNotFoundError with a message when nothing matches.
|
||||
"""
|
||||
unified = home / "logs" / "session" / "unified"
|
||||
if session_arg:
|
||||
candidate = pathlib.Path(session_arg).expanduser()
|
||||
if not candidate.is_dir():
|
||||
candidate = unified / session_arg
|
||||
if not candidate.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"session not found: {session_arg} "
|
||||
f"(looked under {unified})"
|
||||
)
|
||||
return candidate
|
||||
journals = sorted(
|
||||
unified.glob("*/journal/*.jsonl"), key=lambda p: p.stat().st_mtime
|
||||
)
|
||||
if not journals:
|
||||
raise FileNotFoundError(f"no session journals under {unified}")
|
||||
return journals[-1].parent.parent
|
||||
|
||||
|
||||
def read_meta(session_dir):
|
||||
"""Return (title, model_name, alias) from meta.json.
|
||||
|
||||
Unknown fields come back as None; a missing or unreadable
|
||||
meta.json yields (None, None, None).
|
||||
"""
|
||||
meta_path = session_dir / "meta.json"
|
||||
if not meta_path.is_file():
|
||||
return None, None, None
|
||||
try:
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
LOG.warning("meta.json unreadable: %s", exc)
|
||||
return None, None, None
|
||||
config = meta.get("config") or {}
|
||||
return (
|
||||
meta.get("title"),
|
||||
config.get("model_name"),
|
||||
config.get("active_model"),
|
||||
)
|
||||
|
||||
|
||||
def read_projection_usage(session_dir):
|
||||
"""Return the cumulative session tokenUsage dict from the
|
||||
newest projection-state.json, or None when absent.
|
||||
|
||||
The projection state survives journal rotation (Vibe prunes
|
||||
old journal files), so its tokenUsage counter is the
|
||||
authoritative session total. Journal sums only cover the
|
||||
current window.
|
||||
"""
|
||||
states = sorted(
|
||||
session_dir.glob("generations/*/projection-state.json"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
)
|
||||
if not states:
|
||||
return None
|
||||
try:
|
||||
snapshot = json.loads(
|
||||
states[-1].read_text(encoding="utf-8")
|
||||
)["snapshot"]["session"]
|
||||
except (OSError, KeyError, json.JSONDecodeError) as exc:
|
||||
LOG.warning("projection-state.json unreadable: %s", exc)
|
||||
return None
|
||||
usage = snapshot.get("tokenUsage")
|
||||
if not usage:
|
||||
return None
|
||||
cached = min(usage.get("cachedInputTokens") or 0,
|
||||
usage.get("inputTokens") or 0)
|
||||
return {
|
||||
"input": usage.get("inputTokens") or 0,
|
||||
"cached": cached,
|
||||
"output": usage.get("outputTokens") or 0,
|
||||
}
|
||||
|
||||
|
||||
def parse_journals(session_dir):
|
||||
"""Sum usage records from the session's journal files.
|
||||
|
||||
Returns (totals, turns) as plain dicts. Usage records are
|
||||
deduplicated by action_id across journal files. Cached input
|
||||
is clamped to input tokens, matching Vibe's
|
||||
session_token_cost. Turn buckets carry the same keys plus
|
||||
"label".
|
||||
"""
|
||||
totals = {"input": 0, "cached": 0, "output": 0, "calls": 0}
|
||||
turns = []
|
||||
current = {"label": "(before first user message)", "input": 0,
|
||||
"cached": 0, "output": 0, "calls": 0}
|
||||
turns.append(current)
|
||||
seen = set()
|
||||
journal_dir = session_dir / "journal"
|
||||
for journal_path in sorted(journal_dir.glob("*.jsonl")):
|
||||
with journal_path.open(encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
LOG.warning("skipping malformed line in %s",
|
||||
journal_path.name)
|
||||
continue
|
||||
kind = record.get("type")
|
||||
if kind == "projection_delta":
|
||||
for delta in record["payload"].get("delta", []):
|
||||
entry = delta.get("entry") or {}
|
||||
if (entry.get("type") == "message"
|
||||
and entry.get("role") == "user"
|
||||
and entry.get("source") == "turn_start"):
|
||||
text = ""
|
||||
content = entry.get("content") or [{}]
|
||||
if content and isinstance(content[0], dict):
|
||||
text = content[0].get("text", "")
|
||||
current = {"label": text[:60], "input": 0,
|
||||
"cached": 0, "output": 0,
|
||||
"calls": 0}
|
||||
turns.append(current)
|
||||
elif kind == "action_result":
|
||||
action_id = record.get("payload", {}).get("action_id")
|
||||
if action_id in seen:
|
||||
continue
|
||||
result = record["payload"].get("result", {})
|
||||
result = result.get("result", {}) or {}
|
||||
usage = result.get("usage")
|
||||
if not usage:
|
||||
continue
|
||||
seen.add(action_id)
|
||||
for bucket in (totals, current):
|
||||
bucket["input"] += usage.get("input_tokens") or 0
|
||||
bucket["cached"] += (
|
||||
usage.get("cached_input_tokens") or 0
|
||||
)
|
||||
bucket["output"] += usage.get("output_tokens") or 0
|
||||
bucket["calls"] += 1
|
||||
for bucket in turns:
|
||||
bucket["cached"] = min(bucket["cached"], bucket["input"])
|
||||
totals["cached"] = min(totals["cached"], totals["input"])
|
||||
return totals, turns
|
||||
|
||||
|
||||
class VibeHarness(HarnessAdapter):
|
||||
"""Mistral Vibe (Unified Harness) session logs."""
|
||||
|
||||
name = "vibe"
|
||||
home_env_var = "VIBE_HOME"
|
||||
home_dir_name = ".vibe"
|
||||
|
||||
def resolve_session_dir(self, session_arg, home):
|
||||
return resolve_session_dir(session_arg, home)
|
||||
|
||||
def read_meta(self, session_dir):
|
||||
title, model_name, alias = read_meta(session_dir)
|
||||
return SessionMeta(title=title, model_name=model_name,
|
||||
alias=alias)
|
||||
|
||||
def collect_usage(self, session_dir):
|
||||
totals, turns = parse_journals(session_dir)
|
||||
projection = read_projection_usage(session_dir)
|
||||
if projection:
|
||||
input_tokens = projection["input"]
|
||||
cached = projection["cached"]
|
||||
output = projection["output"]
|
||||
source = "projection-state"
|
||||
else:
|
||||
input_tokens = totals["input"]
|
||||
cached = totals["cached"]
|
||||
output = totals["output"]
|
||||
source = "journal"
|
||||
return SessionUsage(
|
||||
input_tokens=input_tokens,
|
||||
cached_input_tokens=min(cached, input_tokens),
|
||||
output_tokens=output,
|
||||
calls=totals["calls"],
|
||||
usage_source=source,
|
||||
turns=[
|
||||
TurnUsage(
|
||||
label=t["label"], calls=t["calls"],
|
||||
input_tokens=t["input"],
|
||||
cached_input_tokens=t["cached"],
|
||||
output_tokens=t["output"],
|
||||
)
|
||||
for t in turns
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"currency": "USD",
|
||||
"models": {
|
||||
"default": {
|
||||
"input_price": 1.5,
|
||||
"cached_input_price": 0.15,
|
||||
"output_price": 7.5
|
||||
},
|
||||
"zai-glm-5-3": {
|
||||
"input_price": 1.5,
|
||||
"cached_input_price": 0.15,
|
||||
"output_price": 7.5,
|
||||
"note": "placeholder at Vibe's built-in default rates; replace with GLM-5.3's real rates when known"
|
||||
},
|
||||
"mistral-medium-3.5": {
|
||||
"input_price": 1.5,
|
||||
"cached_input_price": 0.15,
|
||||
"output_price": 7.5,
|
||||
"note": "Vibe's built-in hosted default model rates"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Pricing table loading, lookup, and updates.
|
||||
|
||||
Prices live in a JSON file (default: pricing.json bundled beside
|
||||
this module) keyed by model, with a "default" fallback entry.
|
||||
All prices are USD per million tokens.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import pathlib
|
||||
|
||||
LOG = logging.getLogger("agent_cost_estimator.pricing")
|
||||
|
||||
DEFAULT_PRICING_PATH = pathlib.Path(__file__).with_name("pricing.json")
|
||||
RATES_KEYS = ("input_price", "cached_input_price", "output_price")
|
||||
|
||||
|
||||
def load_pricing(path=None):
|
||||
"""Load and validate a pricing file.
|
||||
|
||||
Returns the parsed dict: {"currency": "USD", "models": {...}}.
|
||||
Raises FileNotFoundError when the file is missing and ValueError
|
||||
when it is invalid JSON, has no models table, or a model entry
|
||||
lacks a non-negative numeric price.
|
||||
"""
|
||||
path = pathlib.Path(path) if path else DEFAULT_PRICING_PATH
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"pricing file not found: {path}")
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"invalid JSON in {path}: {exc}") from exc
|
||||
models = data.get("models")
|
||||
if not isinstance(models, dict):
|
||||
raise TypeError(f'"models" in {path} is not an object')
|
||||
if not models:
|
||||
raise ValueError(f'no "models" table in {path}')
|
||||
for name, rates in models.items():
|
||||
if not isinstance(rates, dict):
|
||||
raise TypeError(
|
||||
f"entry {name!r} in {path} is not an object"
|
||||
)
|
||||
for key in RATES_KEYS:
|
||||
value = rates.get(key)
|
||||
if not isinstance(value, (int, float)) or value < 0:
|
||||
raise ValueError(
|
||||
f"entry {name!r} in {path} needs a non-negative "
|
||||
f"numeric {key}"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
def get_model_prices(pricing, model_name=None, alias=None):
|
||||
"""Return (rates dict, matched key) for the model.
|
||||
|
||||
Lookup order: model_name, alias, then the "default" entry.
|
||||
Raises ValueError when nothing matches and there is no
|
||||
"default".
|
||||
"""
|
||||
models = pricing["models"]
|
||||
for key in (model_name, alias):
|
||||
if key and key in models:
|
||||
return dict(models[key]), key
|
||||
if "default" in models:
|
||||
return dict(models["default"]), "default"
|
||||
raise ValueError(
|
||||
f"no pricing for model {model_name!r} or {alias!r} and no "
|
||||
'"default" entry'
|
||||
)
|
||||
|
||||
|
||||
def update_pricing(path, model, rates, dry_run=False):
|
||||
"""Write rates for one model into the pricing file.
|
||||
|
||||
Creates the file when missing. Extra keys already stored for
|
||||
the model (notes, sources) are kept. Returns (path, updated)
|
||||
where updated is False when dry_run stopped the write.
|
||||
"""
|
||||
path = pathlib.Path(path)
|
||||
if path.is_file():
|
||||
pricing = load_pricing(path)
|
||||
else:
|
||||
pricing = {"currency": "USD", "models": {}}
|
||||
models = pricing.setdefault("models", {})
|
||||
merged = dict(models.get(model) or {})
|
||||
merged.update(rates)
|
||||
models[model] = merged
|
||||
if dry_run:
|
||||
LOG.info("dry run: would write %s to %s", rates, path)
|
||||
return path, False
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(pricing, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
return path, True
|
||||
@@ -0,0 +1,114 @@
|
||||
---
|
||||
name: agent-cost-estimator
|
||||
version: 0.1.0
|
||||
description: Estimate USD cost of AI agent sessions; run "agent-cost-estimator --skill" for usage.
|
||||
---
|
||||
|
||||
# agent-cost-estimator
|
||||
|
||||
Report token usage and an estimated USD cost for an AI agent
|
||||
session, read from the harness's local logs. Ships with the
|
||||
`vibe` harness; other harnesses plug in as adapters.
|
||||
|
||||
Run `agent-cost-estimator --skill` at any time to re-read these
|
||||
instructions. Everything below assumes the installed
|
||||
`agent-cost-estimator` command; in-tree, use
|
||||
`python -m agent_cost_estimator` from the project folder
|
||||
instead.
|
||||
|
||||
## Quick answers
|
||||
|
||||
```text
|
||||
agent-cost-estimator
|
||||
-> session: $2.11, turn: $2.50
|
||||
(whole session cost, latest turn cost; USD, 2 decimals)
|
||||
|
||||
agent-cost-estimator -s -> 2.1148 (bare session cost)
|
||||
agent-cost-estimator -t -> 2.4979 (bare latest-turn cost)
|
||||
agent-cost-estimator -d -> full report table
|
||||
agent-cost-estimator --json -> one JSON object, all numbers
|
||||
```
|
||||
|
||||
The default line answers "what does this session cost and what
|
||||
did the last turn cost". `-s`/`-t` print bare numbers for
|
||||
scripting; `-d` shows the table with token counts, pricing
|
||||
source, and caveats; `--json` is the full machine-readable
|
||||
report (tokens, cost breakdown, per-turn list, pricing).
|
||||
|
||||
## What to trust
|
||||
|
||||
- Session-wide token totals come from the harness's cumulative
|
||||
counter (for vibe: `projection-state.json`), which survives
|
||||
log rotation.
|
||||
- Call counts and per-turn breakdowns come from journal files,
|
||||
which the harness prunes: they cover only the window on disk.
|
||||
The latest turn is always in that window, so `-t` is safe.
|
||||
- All costs are **estimates** at the configured rates, never
|
||||
billed figures. Tell the user that, and point them at their
|
||||
provider's console (Mistral: admin.mistral.ai/subscription)
|
||||
for authoritative spend.
|
||||
|
||||
## Selecting what to measure
|
||||
|
||||
```text
|
||||
--harness NAME adapter to read (default: vibe; list: --list-harnesses)
|
||||
--session-id ID|PATH a specific session id or directory
|
||||
--home PATH the harness's data root (default: harness-specific,
|
||||
vibe: $VIBE_HOME or ~/.vibe)
|
||||
```
|
||||
|
||||
With no `--session-id`, the most recently active session is
|
||||
picked (newest journal file). To inspect another session, pass
|
||||
its id or path.
|
||||
|
||||
## Pricing
|
||||
|
||||
Prices live in a JSON table, USD per million tokens, keyed by
|
||||
model with a `default` fallback. Lookup order: the session's
|
||||
model name, its alias, then `default`.
|
||||
|
||||
```text
|
||||
--pricing FILE use a custom pricing table
|
||||
--model KEY force the lookup key
|
||||
--input-price P --cached-input-price P --output-price P
|
||||
override for one run
|
||||
--update-pricing [--dry-run] persist the overrides
|
||||
(requires --pricing or the bundled file)
|
||||
```
|
||||
|
||||
To store real rates once: run
|
||||
`agent-cost-estimator --model <model> --input-price P --cached-input-price P --output-price P --update-pricing`.
|
||||
Later runs then price correctly without flags.
|
||||
|
||||
## Reading output correctly
|
||||
|
||||
- `turn:` / `-t` is the **latest turn**, priced at the same
|
||||
rates. While a turn is still running, it counts only what has
|
||||
been flushed to the journal so far (an under-estimate).
|
||||
- Cached input is clamped to input tokens; fresh input is
|
||||
`input - cached`.
|
||||
- With `--update-pricing`, the report still prints normally
|
||||
after the stderr confirmation line; parse stdout only.
|
||||
|
||||
## Extending for other harnesses
|
||||
|
||||
A harness adapter subclasses
|
||||
`agent_cost_estimator.harnesses.base.HarnessAdapter` and
|
||||
implements `resolve_session_dir`, `read_meta`, and
|
||||
`collect_usage`, then registers through the
|
||||
`agent_cost_estimator.harnesses` entry-point group:
|
||||
|
||||
```toml
|
||||
[project.entry-points."agent_cost_estimator.harnesses"]
|
||||
mytool = "my_package.adapters:MyToolHarness"
|
||||
```
|
||||
|
||||
Library use: `from agent_cost_estimator import
|
||||
available_harnesses, get_harness, compute_cost, load_pricing`.
|
||||
|
||||
## Errors
|
||||
|
||||
Exit code 1 with `error: ...` on stderr: unknown session, no
|
||||
sessions under the home, unknown `--harness`, or a missing or
|
||||
invalid pricing file. `--skill` and `--list-harnesses` never
|
||||
touch session data and always work.
|
||||
@@ -0,0 +1,2 @@
|
||||
# This directory contains the SKILL.md file for LLM consumption
|
||||
# via the --skill flag
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# Build script for agent-cost-estimator project
|
||||
# Creates source and wheel distributions
|
||||
|
||||
set -e
|
||||
|
||||
# Check if build module is available, install if not
|
||||
if ! python -m build --version >/dev/null 2>&1; then
|
||||
echo "Installing build package..."
|
||||
python -m pip install build --break-system-packages
|
||||
fi
|
||||
|
||||
# Build the distribution files
|
||||
rm -rf dist/ build/ agent_cost_estimator.egg-info/
|
||||
|
||||
python -m build
|
||||
|
||||
echo "Build completed. Distribution files are in the dist/ directory."
|
||||
echo "To upload to PyPI: twine upload dist/*"
|
||||
@@ -0,0 +1,40 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=77"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "agent-cost-estimator"
|
||||
version = "0.1.0"
|
||||
description = "Report token usage and estimated USD cost of AI agent sessions from local harness logs"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
license = "MIT"
|
||||
license-files = ["LICENSE"]
|
||||
authors = [{ name = "nvaldez" }]
|
||||
keywords = ["ai", "agents", "llm", "tokens", "cost", "cli", "mistral", "vibe"]
|
||||
classifiers = [
|
||||
"Development Status :: 4 - Beta",
|
||||
"Environment :: Console",
|
||||
"Intended Audience :: Developers",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
agent-cost-estimator = "agent_cost_estimator.cli:main"
|
||||
|
||||
[project.entry-points."agent_cost_estimator.harnesses"]
|
||||
vibe = "agent_cost_estimator.harnesses.vibe:VibeHarness"
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["agent_cost_estimator", "agent_cost_estimator.harnesses", "agent_cost_estimator.skill"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
agent_cost_estimator = ["pricing.json"]
|
||||
skill = ["SKILL.md"]
|
||||
Executable
+510
@@ -0,0 +1,510 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for agent_cost_estimator.
|
||||
|
||||
Covers: cost math, pricing table loading/lookup/update, the vibe
|
||||
harness adapter (journal parsing, cached clamp, dedupe, per-turn
|
||||
grouping, projection-state preference, session discovery), the
|
||||
harness registry, and the CLI (pricing flags, -t, --json,
|
||||
--update-pricing, error exits).
|
||||
"""
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, str(pathlib.Path(__file__).parent.parent)
|
||||
) # run against the in-tree package
|
||||
|
||||
from agent_cost_estimator import cli, core, pricing
|
||||
from agent_cost_estimator.harnesses import (
|
||||
available_harnesses,
|
||||
get_harness,
|
||||
)
|
||||
from agent_cost_estimator.harnesses import vibe as vh
|
||||
|
||||
# --- fixtures: a fake vibe session on disk ---
|
||||
|
||||
def write_journal(session_dir, lines, mtime=None):
|
||||
"""Write one journal file; optionally set its mtime."""
|
||||
journal = session_dir / "journal"
|
||||
journal.mkdir(parents=True, exist_ok=True)
|
||||
path = journal / "0000000000000001.jsonl"
|
||||
path.write_text("\n".join(json.dumps(line) for line in lines) + "\n",
|
||||
encoding="utf-8")
|
||||
if mtime is not None:
|
||||
import os
|
||||
os.utime(path, (mtime, mtime))
|
||||
return path
|
||||
|
||||
|
||||
def usage_result(action_id, inp, cached, out):
|
||||
return {
|
||||
"type": "action_result",
|
||||
"payload": {
|
||||
"action_id": action_id,
|
||||
"result": {"result": {"usage": {
|
||||
"input_tokens": inp,
|
||||
"cached_input_tokens": cached,
|
||||
"output_tokens": out,
|
||||
}}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def tool_result(action_id):
|
||||
"""An action_result without usage (hook/tool) - must be ignored."""
|
||||
return {
|
||||
"type": "action_result",
|
||||
"payload": {
|
||||
"action_id": action_id,
|
||||
"result": {"result": {"hook": "post_tool_call",
|
||||
"output": {}}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def user_message(text):
|
||||
return {
|
||||
"type": "projection_delta",
|
||||
"payload": {"delta": [{"op": "append_entry", "entry": {
|
||||
"type": "message", "role": "user", "source": "turn_start",
|
||||
"content": [{"type": "text", "text": text}],
|
||||
}}]},
|
||||
}
|
||||
|
||||
|
||||
def write_projection(session_dir, inp, cached, out):
|
||||
gen = session_dir / "generations" / "0000000000000001"
|
||||
gen.mkdir(parents=True, exist_ok=True)
|
||||
(gen / "projection-state.json").write_text(json.dumps({
|
||||
"snapshot": {"session": {"tokenUsage": {
|
||||
"inputTokens": inp, "cachedInputTokens": cached,
|
||||
"outputTokens": out,
|
||||
}}}},
|
||||
), encoding="utf-8")
|
||||
|
||||
|
||||
def write_meta(session_dir, title=None, model_name=None, alias=None):
|
||||
(session_dir / "meta.json").write_text(json.dumps({
|
||||
"title": title, "config": {"model_name": model_name,
|
||||
"active_model": alias}}),
|
||||
encoding="utf-8")
|
||||
|
||||
|
||||
def write_pricing_file(path, models=None, currency="USD"):
|
||||
path.write_text(json.dumps({
|
||||
"currency": currency,
|
||||
"models": models or {
|
||||
"default": {"input_price": 1.5, "cached_input_price": 0.15,
|
||||
"output_price": 7.5}},
|
||||
}), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
# --- core ---
|
||||
|
||||
def test_compute_cost():
|
||||
"""Cost math splits fresh, cached, and output tokens."""
|
||||
cost = core.compute_cost(
|
||||
{"fresh": 700, "cached": 2300, "output": 150},
|
||||
{"input_price": 1.5, "cached_input_price": 0.15,
|
||||
"output_price": 7.5})
|
||||
assert cost["total_usd"] == round(
|
||||
(700 * 1.5 + 2300 * 0.15 + 150 * 7.5) / 1e6, 4)
|
||||
|
||||
|
||||
# --- pricing ---
|
||||
|
||||
def test_bundled_pricing_loads():
|
||||
"""The shipped pricing.json is valid and has a default."""
|
||||
data = pricing.load_pricing()
|
||||
assert data["currency"] == "USD"
|
||||
assert "default" in data["models"]
|
||||
|
||||
|
||||
def test_pricing_lookup_order(tmp_path):
|
||||
"""model_name wins, then alias, then default."""
|
||||
path = write_pricing_file(tmp_path / "p.json", models={
|
||||
"default": {"input_price": 1, "cached_input_price": 0.1,
|
||||
"output_price": 5},
|
||||
"glm-5-3": {"input_price": 2, "cached_input_price": 0.2,
|
||||
"output_price": 6},
|
||||
"zai-glm-5-3": {"input_price": 3, "cached_input_price": 0.3,
|
||||
"output_price": 7},
|
||||
})
|
||||
data = pricing.load_pricing(path)
|
||||
rates, key = pricing.get_model_prices(data, "zai-glm-5-3", "glm-5-3")
|
||||
assert key == "zai-glm-5-3"
|
||||
assert rates["input_price"] == 3
|
||||
rates, key = pricing.get_model_prices(data, None, "glm-5-3")
|
||||
assert key == "glm-5-3"
|
||||
rates, key = pricing.get_model_prices(data, "unknown", "also-unknown")
|
||||
assert key == "default"
|
||||
assert rates["input_price"] == 1
|
||||
|
||||
|
||||
def test_pricing_validation(tmp_path):
|
||||
"""Invalid pricing files raise TypeError/ValueError."""
|
||||
bad = tmp_path / "bad.json"
|
||||
bad.write_text("{ not json", encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
pricing.load_pricing(bad)
|
||||
wrong_type = tmp_path / "wt.json"
|
||||
wrong_type.write_text(json.dumps({"models": []}), encoding="utf-8")
|
||||
with pytest.raises(TypeError):
|
||||
pricing.load_pricing(wrong_type)
|
||||
no_models = tmp_path / "nm.json"
|
||||
no_models.write_text(json.dumps({"models": {}}), encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
pricing.load_pricing(no_models)
|
||||
negative = tmp_path / "neg.json"
|
||||
write_pricing_file(negative, models={
|
||||
"default": {"input_price": -1, "cached_input_price": 0.1,
|
||||
"output_price": 5}})
|
||||
with pytest.raises(ValueError):
|
||||
pricing.load_pricing(negative)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
pricing.load_pricing(tmp_path / "missing.json")
|
||||
|
||||
|
||||
def test_update_pricing_writes_and_preserves(tmp_path):
|
||||
"""update_pricing writes rates, keeps notes, honors dry_run."""
|
||||
path = write_pricing_file(tmp_path / "p.json", models={
|
||||
"default": {"input_price": 1, "cached_input_price": 0.1,
|
||||
"output_price": 5},
|
||||
"zai-glm-5-3": {"input_price": 1, "cached_input_price": 0.1,
|
||||
"output_price": 5, "note": "keep me"},
|
||||
})
|
||||
pricing.update_pricing(path, "zai-glm-5-3",
|
||||
{"input_price": 3, "cached_input_price": 0.3,
|
||||
"output_price": 7})
|
||||
entry = pricing.load_pricing(path)["models"]["zai-glm-5-3"]
|
||||
assert entry["input_price"] == 3
|
||||
assert entry["note"] == "keep me"
|
||||
# dry run leaves the file unchanged
|
||||
before = path.read_text()
|
||||
_path, written = pricing.update_pricing(
|
||||
path, "new-model", {"input_price": 1, "cached_input_price": 0.1,
|
||||
"output_price": 5}, dry_run=True)
|
||||
assert written is False
|
||||
assert path.read_text() == before
|
||||
# creating a new file from scratch works
|
||||
fresh = tmp_path / "fresh" / "p.json"
|
||||
pricing.update_pricing(fresh, "m", {"input_price": 1,
|
||||
"cached_input_price": 0.1,
|
||||
"output_price": 5})
|
||||
assert "m" in pricing.load_pricing(fresh)["models"]
|
||||
|
||||
|
||||
# --- harness registry ---
|
||||
|
||||
def test_registry_has_vibe_and_rejects_unknown():
|
||||
"""vibe is registered; unknown names raise ValueError."""
|
||||
assert "vibe" in available_harnesses()
|
||||
harness = get_harness("vibe")
|
||||
assert harness.name == "vibe"
|
||||
with pytest.raises(ValueError):
|
||||
get_harness("not-a-harness")
|
||||
|
||||
|
||||
def test_list_harnesses_flag(capsys):
|
||||
"""--list-harnesses prints the registry, one name per line."""
|
||||
code = cli.main(["--list-harnesses"])
|
||||
assert code == 0
|
||||
out = capsys.readouterr().out.split()
|
||||
assert "vibe" in out
|
||||
|
||||
|
||||
def test_skill_flag_prints_the_skill_doc(capsys):
|
||||
"""--skill prints SKILL.md verbatim, without reading a session."""
|
||||
code = cli.main(["--skill"])
|
||||
assert code == 0
|
||||
out = capsys.readouterr().out
|
||||
text = cli.SKILL_PATH.read_text(encoding="utf-8")
|
||||
assert out == text
|
||||
assert "agent-cost-estimator" in out
|
||||
assert "session-wide" in out or "Quick answers" in out
|
||||
|
||||
|
||||
# --- vibe adapter ---
|
||||
|
||||
def test_vibe_journal_totals(tmp_path):
|
||||
"""Input, cached, and output tokens sum across calls."""
|
||||
session = tmp_path / "s1"
|
||||
write_journal(session, [
|
||||
usage_result("a1", 1000, 800, 50),
|
||||
usage_result("a2", 2000, 1500, 100),
|
||||
tool_result("a3"),
|
||||
])
|
||||
totals, _turns = vh.parse_journals(session)
|
||||
assert totals["calls"] == 2
|
||||
assert totals["input"] == 3000
|
||||
assert totals["cached"] == 2300
|
||||
assert totals["output"] == 150
|
||||
|
||||
|
||||
def test_vibe_dedupe_by_action_id(tmp_path):
|
||||
"""The same action_id in two journal files counts once."""
|
||||
session = tmp_path / "s1"
|
||||
journal = session / "journal"
|
||||
journal.mkdir(parents=True)
|
||||
line = json.dumps(usage_result("a1", 1000, 0, 10))
|
||||
for name in ("0000000000000001.jsonl", "0000000000000002.jsonl"):
|
||||
(journal / name).write_text(line + "\n", encoding="utf-8")
|
||||
totals, _turns = vh.parse_journals(session)
|
||||
assert totals["calls"] == 1
|
||||
assert totals["input"] == 1000
|
||||
|
||||
|
||||
def test_vibe_cached_clamped_to_input(tmp_path):
|
||||
"""Cached over-counting never yields negative fresh tokens."""
|
||||
session = tmp_path / "s1"
|
||||
write_journal(session, [usage_result("a1", 100, 150, 5)])
|
||||
usage = vh.VibeHarness().collect_usage(session)
|
||||
assert usage.cached_input_tokens == 100
|
||||
assert usage.fresh_input_tokens == 0
|
||||
|
||||
|
||||
def test_vibe_per_turn_grouping(tmp_path):
|
||||
"""Usage records fall into the turn their call belongs to."""
|
||||
session = tmp_path / "s1"
|
||||
write_journal(session, [
|
||||
user_message("first question"),
|
||||
usage_result("a1", 1000, 500, 20),
|
||||
user_message("second question"),
|
||||
usage_result("a2", 2000, 1000, 40),
|
||||
usage_result("a3", 3000, 1500, 60),
|
||||
])
|
||||
usage = vh.VibeHarness().collect_usage(session)
|
||||
used = [t for t in usage.turns if t.calls]
|
||||
assert len(used) == 2
|
||||
assert used[0].label == "first question"
|
||||
assert used[0].input_tokens == 1000
|
||||
assert used[1].input_tokens == 5000
|
||||
assert used[1].calls == 2
|
||||
|
||||
|
||||
def test_vibe_projection_preferred(tmp_path):
|
||||
"""Projection-state totals override journal sums, which lose
|
||||
pruned history to journal rotation."""
|
||||
session = tmp_path / "s1"
|
||||
write_journal(session, [usage_result("a1", 1000, 500, 50)])
|
||||
write_projection(session, 5000, 4000, 200)
|
||||
usage = vh.VibeHarness().collect_usage(session)
|
||||
assert usage.usage_source == "projection-state"
|
||||
assert usage.input_tokens == 5000
|
||||
assert usage.cached_input_tokens == 4000
|
||||
assert usage.fresh_input_tokens == 1000
|
||||
assert usage.output_tokens == 200
|
||||
assert usage.calls == 1 # journal window
|
||||
|
||||
|
||||
def test_vibe_fallback_to_journal_totals(tmp_path):
|
||||
"""Without projection-state.json, journal sums are the totals."""
|
||||
session = tmp_path / "s1"
|
||||
write_journal(session, [usage_result("a1", 1000, 500, 50)])
|
||||
usage = vh.VibeHarness().collect_usage(session)
|
||||
assert usage.usage_source == "journal"
|
||||
assert usage.input_tokens == 1000
|
||||
assert usage.output_tokens == 50
|
||||
|
||||
|
||||
def test_vibe_session_discovery(tmp_path):
|
||||
"""Without --session, the newest journal wins; ids resolve too."""
|
||||
unified = tmp_path / "logs" / "session" / "unified"
|
||||
old = unified / "old-session"
|
||||
new = unified / "new-session"
|
||||
write_journal(old, [usage_result("a1", 1, 0, 1)], mtime=time.time() - 100)
|
||||
write_journal(new, [usage_result("a2", 2, 0, 1)], mtime=time.time())
|
||||
harness = vh.VibeHarness()
|
||||
picked = harness.resolve_session_dir(None, tmp_path)
|
||||
assert picked.name == "new-session"
|
||||
by_id = harness.resolve_session_dir("old-session", tmp_path)
|
||||
assert by_id.name == "old-session"
|
||||
with pytest.raises(FileNotFoundError):
|
||||
harness.resolve_session_dir("nope", tmp_path)
|
||||
|
||||
|
||||
def test_vibe_read_meta(tmp_path):
|
||||
"""read_meta returns title and pricing keys."""
|
||||
session = tmp_path / "s1"
|
||||
session.mkdir()
|
||||
write_meta(session, title="t", model_name="m1", alias="m1a")
|
||||
meta = vh.VibeHarness().read_meta(session)
|
||||
assert meta.title == "t"
|
||||
assert meta.model_name == "m1"
|
||||
assert meta.alias == "m1a"
|
||||
|
||||
|
||||
def test_vibe_default_home():
|
||||
"""default_home prefers the env var, then ~/.<home_dir_name>."""
|
||||
harness = vh.VibeHarness()
|
||||
home = harness.default_home()
|
||||
assert home == pathlib.Path.home() / ".vibe" or home.name == ".vibe"
|
||||
|
||||
|
||||
# --- cli ---
|
||||
|
||||
def test_cli_json_with_custom_pricing(tmp_path, capsys):
|
||||
"""--json with a custom pricing file prices the session."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [
|
||||
user_message("hi"),
|
||||
usage_result("a1", 1000, 500, 50),
|
||||
])
|
||||
write_projection(session, 1000, 500, 50)
|
||||
write_meta(session, title="t", model_name="m1", alias="m1a")
|
||||
pfile = write_pricing_file(tmp_path / "p.json", models={
|
||||
"default": {"input_price": 1, "cached_input_price": 0.1,
|
||||
"output_price": 5},
|
||||
"m1": {"input_price": 2, "cached_input_price": 0.2,
|
||||
"output_price": 6}})
|
||||
code = cli.main(["--session-id", str(session), "--home", str(tmp_path),
|
||||
"--pricing", str(pfile), "--json"])
|
||||
assert code == 0
|
||||
data = json.loads(capsys.readouterr().out)
|
||||
assert data["harness"] == "vibe"
|
||||
assert data["pricing"]["model_key"] == "m1"
|
||||
assert data["usage_source"] == "projection-state"
|
||||
assert data["cost_usd"]["total_usd"] == round(
|
||||
(500 * 2 + 500 * 0.2 + 50 * 6) / 1e6, 4)
|
||||
|
||||
|
||||
def test_cli_session_flag_prints_only_the_number(tmp_path, capsys):
|
||||
"""-s prints just the session cost, nothing else."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [usage_result("a1", 1000, 500, 50)])
|
||||
code = cli.main(["--session-id", str(session), "--home",
|
||||
str(tmp_path), "-s"])
|
||||
assert code == 0
|
||||
out = capsys.readouterr().out.strip()
|
||||
assert out == f"{round((500 * 1.5 + 500 * 0.15 + 50 * 7.5) / 1e6, 4):.4f}"
|
||||
|
||||
|
||||
def test_cli_turn_flag_prints_only_the_turn(tmp_path, capsys):
|
||||
"""-t prints just the latest turn's cost."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [
|
||||
user_message("first"),
|
||||
usage_result("a1", 1000, 500, 50),
|
||||
user_message("second"),
|
||||
usage_result("a2", 3000, 2000, 400),
|
||||
])
|
||||
code = cli.main(["--session-id", str(session), "--home",
|
||||
str(tmp_path), "-t"])
|
||||
assert code == 0
|
||||
out = capsys.readouterr().out.strip()
|
||||
# only the second turn's usage, not the session total
|
||||
assert out == f"{round((1000 * 1.5 + 2000 * 0.15 + 400 * 7.5) / 1e6, 4):.4f}"
|
||||
|
||||
|
||||
def test_cli_default_compact_line(tmp_path, capsys):
|
||||
"""No output flag: one line with session and turn costs."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [
|
||||
user_message("first"),
|
||||
usage_result("a1", 1_000_000, 0, 0),
|
||||
user_message("second"),
|
||||
usage_result("a2", 1_000_000, 500_000, 100_000),
|
||||
])
|
||||
code = cli.main(["--session-id", str(session), "--home",
|
||||
str(tmp_path)])
|
||||
assert code == 0
|
||||
out = capsys.readouterr().out.strip()
|
||||
# session: first+second turn usage; turn: second turn only
|
||||
session_cost = (1_000_000 * 1.5 + 500_000 * 1.5
|
||||
+ 500_000 * 0.15 + 100_000 * 7.5) / 1e6
|
||||
turn_cost = (500_000 * 1.5 + 500_000 * 0.15
|
||||
+ 100_000 * 7.5) / 1e6
|
||||
assert out == (f"session: ${session_cost:.2f}, "
|
||||
f"turn: ${turn_cost:.2f}")
|
||||
|
||||
|
||||
def test_cli_details_shows_the_full_table(tmp_path, capsys):
|
||||
"""-d prints the full report table."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [usage_result("a1", 1000, 500, 50)])
|
||||
write_meta(session, title="t", model_name="m1")
|
||||
code = cli.main(["--session-id", str(session), "--home",
|
||||
str(tmp_path), "--details"])
|
||||
assert code == 0
|
||||
out = capsys.readouterr().out
|
||||
assert out.startswith("harness: vibe")
|
||||
assert "TOTAL:" in out
|
||||
assert "usage source:" in out
|
||||
|
||||
|
||||
def test_cli_price_overrides_beat_pricing_file(tmp_path, capsys):
|
||||
"""CLI price flags override the pricing file for the run."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [usage_result("a1", 1000, 500, 50)])
|
||||
code = cli.main(["--session-id", str(session), "--home", str(tmp_path),
|
||||
"-s", "--input-price", "1",
|
||||
"--output-price", "10"])
|
||||
assert code == 0
|
||||
out = capsys.readouterr().out.strip()
|
||||
assert out == f"{round((500 * 1 + 500 * 0.15 + 50 * 10) / 1e6, 4):.4f}"
|
||||
|
||||
|
||||
def test_cli_update_pricing_persists(tmp_path, capsys):
|
||||
"""--update-pricing writes the overrides into the file."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [usage_result("a1", 1000, 500, 50)])
|
||||
pfile = write_pricing_file(tmp_path / "p.json")
|
||||
code = cli.main(["--session-id", str(session), "--home", str(tmp_path),
|
||||
"--pricing", str(pfile), "-s",
|
||||
"--model", "zai-glm-5-3", "--input-price", "1.2",
|
||||
"--update-pricing"])
|
||||
assert code == 0
|
||||
capsys.readouterr() # drain: the update run's -s output
|
||||
entry = pricing.load_pricing(pfile)["models"]["zai-glm-5-3"]
|
||||
assert entry["input_price"] == 1.2
|
||||
assert entry["cached_input_price"] == 0.15 # carried from default
|
||||
# a second run now prices from the file without overrides
|
||||
code = cli.main(["--session-id", str(session), "--home", str(tmp_path),
|
||||
"--pricing", str(pfile), "-s",
|
||||
"--model", "zai-glm-5-3"])
|
||||
assert code == 0
|
||||
out = capsys.readouterr().out.strip()
|
||||
assert out == f"{round((500 * 1.2 + 500 * 0.15 + 50 * 7.5) / 1e6, 4):.4f}"
|
||||
|
||||
|
||||
def test_cli_update_pricing_needs_a_price(tmp_path, capsys):
|
||||
"""--update-pricing without any price override is an error."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [usage_result("a1", 1, 0, 1)])
|
||||
code = cli.main(["--session-id", str(session), "--home", str(tmp_path),
|
||||
"--update-pricing"])
|
||||
assert code == 1
|
||||
assert "--update-pricing needs" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_cli_dry_run_needs_update_pricing(tmp_path, capsys):
|
||||
"""--dry-run without --update-pricing is an error."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [usage_result("a1", 1, 0, 1)])
|
||||
code = cli.main(["--session-id", str(session), "--home", str(tmp_path),
|
||||
"--dry-run"])
|
||||
assert code == 1
|
||||
assert "--dry-run needs" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_cli_unknown_harness_exits(tmp_path, capsys):
|
||||
"""An unknown --harness exits 1 with the error on stderr."""
|
||||
session = tmp_path / "logs" / "session" / "unified" / "s1"
|
||||
write_journal(session, [usage_result("a1", 1, 0, 1)])
|
||||
code = cli.main(["--harness", "not-a-harness", "--session-id",
|
||||
str(session), "--home", str(tmp_path)])
|
||||
assert code == 1
|
||||
assert "unknown harness" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_cli_error_exit(tmp_path, capsys):
|
||||
"""An unknown session exits 1 with the error on stderr."""
|
||||
code = cli.main(["--session-id", "nope", "--home", str(tmp_path)])
|
||||
assert code == 1
|
||||
assert "session not found" in capsys.readouterr().err
|
||||
Reference in New Issue
Block a user