94e7534a79
Different credit pools can share one pricing table, so the display key
now reports the pool a session actually bills. Two optional pricing
fields drive it:
- harness_credit_pools ({"vibe": "mistral-vibe", "hermes": "mistral-api"})
for harnesses whose logs do not record the billing API (Vibe session
logs carry only the model name)
- billing_url_pools ({"api.mistral.ai": "mistral-api"}) for harnesses
that do: Hermes logs session_model_usage.billing_base_url, and a URL
match wins over the harness mapping
SessionMeta gains base_url; the Hermes adapter exposes the logged
billing base URL (tolerates older schemas without the column). The pool
overrides the provider part of the display key (mistral-api:mistral-
large-4 for a Hermes session on the pay-as-you-go API, mistral-vibe:
zai-glm-5-3 for a Vibe session), appears as a "credit pool" line in -d
and credit_pool in --json, and the GUI refresh resolver maps
pool-prefixed strings back to the model's real rates. Rates always
come from the matched pricing entry.
Verified live against both harnesses: mistral-vibe:zai-glm-5-3
(Vibe) and mistral-api:mistral-large-4 (Hermes on api.mistral.ai).
Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
476 lines
20 KiB
Python
476 lines
20 KiB
Python
"""Tests for the GUI module."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pathlib
|
|
import tempfile
|
|
import sqlite3
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from agent_cost_estimator.gui import (
|
|
SessionInfo,
|
|
SessionDiscovery,
|
|
Settings,
|
|
get_model_rates_for_display,
|
|
get_settings_path,
|
|
)
|
|
from agent_cost_estimator.core import compute_cost
|
|
from agent_cost_estimator.pricing import load_pricing
|
|
from agent_cost_estimator.harnesses import VibeHarness, HermesHarness
|
|
|
|
|
|
class TestSettings:
|
|
"""Test settings management."""
|
|
|
|
def test_get_settings_path_windows(self):
|
|
"""Test settings path on Windows."""
|
|
with patch("sys.platform", "win32"):
|
|
with patch.dict("os.environ", {"APPDATA": "C:\\Users\\Test\\AppData\\Roaming"}):
|
|
settings_path = get_settings_path()
|
|
# On Linux, pathlib might use forward slashes even for Windows paths
|
|
assert "agent-cost-estimator" in str(settings_path)
|
|
assert "settings.json" in str(settings_path)
|
|
|
|
def test_get_settings_path_linux(self):
|
|
"""Test settings path on Linux."""
|
|
with patch("sys.platform", "linux"):
|
|
# Use a temp directory instead of system paths
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
with patch.dict("os.environ", {"XDG_CONFIG_HOME": tmpdir}):
|
|
settings_path = get_settings_path()
|
|
assert "agent-cost-estimator" in str(settings_path)
|
|
assert "settings.json" in str(settings_path)
|
|
|
|
def test_get_settings_path_linux_default(self):
|
|
"""Test settings path on Linux with default XDG_CONFIG_HOME."""
|
|
with patch("sys.platform", "linux"):
|
|
with patch.dict("os.environ", {}, clear=True):
|
|
with patch("pathlib.Path.home", return_value=pathlib.Path("/home/test")):
|
|
settings_path = get_settings_path()
|
|
assert str(settings_path).endswith(".config/agent-cost-estimator/settings.json")
|
|
|
|
def test_settings_default(self):
|
|
"""Test default settings."""
|
|
settings = Settings()
|
|
assert settings.default_harness == "vibe"
|
|
|
|
def test_settings_load(self):
|
|
"""Test loading settings from file."""
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
|
|
f.write('{"default_harness": "hermes"}')
|
|
f.flush()
|
|
|
|
with patch.object(Settings, "load") as mock_load:
|
|
mock_load.return_value = Settings(default_harness="hermes")
|
|
settings = Settings.load()
|
|
assert settings.default_harness == "hermes"
|
|
|
|
|
|
class TestGetModelRatesForDisplay:
|
|
"""Test pricing lookup from provider-qualified display strings."""
|
|
|
|
def test_qualified_model_resolves_to_real_rates(self):
|
|
"""A "provider:model" string must match the model's own rates,
|
|
not fall back to the global default rates."""
|
|
pricing = load_pricing()
|
|
rates, key, currency, is_default = get_model_rates_for_display(
|
|
pricing, "zai:zai-glm-5-3")
|
|
assert key == "zai:zai-glm-5-3"
|
|
assert not is_default
|
|
assert rates["input_price"] == 1.4
|
|
assert rates["output_price"] == 4.4
|
|
|
|
def test_qualified_model_cost_matches_compute_cost(self):
|
|
"""GUI refresh must price a GLM session at GLM rates, not at
|
|
the Medium-3.5-shaped fallback rates."""
|
|
pricing = load_pricing()
|
|
rates, _, _, _ = get_model_rates_for_display(
|
|
pricing, "zai:zai-glm-5-3")
|
|
tokens = {"fresh": 72_670, "cached": 552_640, "output": 3_344}
|
|
costs = compute_cost(tokens, rates)
|
|
assert costs["total_usd"] == pytest.approx(0.1938, abs=0.0005)
|
|
|
|
def test_aliased_display_resolves_to_real_rates(self):
|
|
"""A provider_alias display key ("mistral:zai-glm-5-3") must
|
|
resolve back to the zai rates, not mistral fallback rates."""
|
|
pricing = load_pricing()
|
|
rates, key, currency, is_default = get_model_rates_for_display(
|
|
pricing, "mistral:zai-glm-5-3")
|
|
assert key == "zai:zai-glm-5-3"
|
|
assert not is_default
|
|
assert rates["input_price"] == 1.4
|
|
assert rates["output_price"] == 4.4
|
|
|
|
def test_pool_display_resolves_to_real_rates(self):
|
|
"""Pool-prefixed display keys ("mistral-vibe:zai-glm-5-3",
|
|
"mistral-api:mistral-large-4") resolve to the model's own
|
|
rates; the pool names a billing pool, not a provider."""
|
|
pricing = load_pricing()
|
|
rates, key, _, _ = get_model_rates_for_display(
|
|
pricing, "mistral-vibe:zai-glm-5-3")
|
|
assert key == "zai:zai-glm-5-3"
|
|
assert rates["output_price"] == 4.4
|
|
rates, key, _, _ = get_model_rates_for_display(
|
|
pricing, "mistral-api:mistral-large-4")
|
|
assert key == "mistral:mistral-large-4"
|
|
assert rates["input_price"] == 0.7
|
|
assert rates["output_price"] == 2.1
|
|
|
|
def test_bare_model_falls_back_to_default(self):
|
|
"""An unqualified unknown model still gets the fallback rates."""
|
|
pricing = load_pricing()
|
|
rates, key, currency, is_default = get_model_rates_for_display(
|
|
pricing, "unknown")
|
|
assert is_default
|
|
assert key.endswith(":default")
|
|
assert rates["input_price"] == 1.5
|
|
|
|
def test_none_model_falls_back_to_default(self):
|
|
"""A missing model string still gets the fallback rates."""
|
|
pricing = load_pricing()
|
|
rates, key, currency, is_default = get_model_rates_for_display(
|
|
pricing, None)
|
|
assert is_default
|
|
assert key.endswith(":default")
|
|
assert rates["output_price"] == 7.5
|
|
|
|
|
|
class TestSessionInfo:
|
|
"""Test SessionInfo dataclass."""
|
|
|
|
def test_session_info_creation(self):
|
|
"""Test creating a SessionInfo object."""
|
|
info = SessionInfo(
|
|
session_id="test-session-id",
|
|
harness_name="vibe",
|
|
title="Test Session",
|
|
model="mistral:mistral-medium",
|
|
session_price=1.50,
|
|
current_turn_price=0.25,
|
|
session_dir=pathlib.Path("/tmp/session"),
|
|
)
|
|
assert info.session_id == "test-session-id"
|
|
assert info.harness_name == "vibe"
|
|
assert info.title == "Test Session"
|
|
assert info.model == "mistral:mistral-medium"
|
|
assert info.session_price == 1.50
|
|
assert info.current_turn_price == 0.25
|
|
|
|
|
|
class TestSessionDiscovery:
|
|
"""Test session discovery."""
|
|
|
|
def test_harness_adapters(self):
|
|
"""Test that harness adapters are registered."""
|
|
discovery = SessionDiscovery(pricing_data={})
|
|
assert "vibe" in discovery.HARNESS_ADAPTERS
|
|
assert "hermes" in discovery.HARNESS_ADAPTERS
|
|
assert discovery.HARNESS_ADAPTERS["vibe"] is VibeHarness
|
|
assert discovery.HARNESS_ADAPTERS["hermes"] is HermesHarness
|
|
|
|
def test_discover_vibe_sessions(self):
|
|
"""Test discovering Vibe sessions."""
|
|
# Create a mock home directory with session structure
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
home = pathlib.Path(tmpdir)
|
|
|
|
# Create Vibe session structure
|
|
unified_dir = home / "logs" / "session" / "unified"
|
|
unified_dir.mkdir(parents=True)
|
|
|
|
session_dir = unified_dir / "test-session-123"
|
|
session_dir.mkdir()
|
|
|
|
# Create meta.json
|
|
import json
|
|
meta = {
|
|
"title": "Test Session",
|
|
"config": {
|
|
"model_name": "mistral-medium",
|
|
"active_model": "mistral-medium"
|
|
}
|
|
}
|
|
(session_dir / "meta.json").write_text(json.dumps(meta))
|
|
|
|
# Create projection-state.json
|
|
projection_state = {
|
|
"tokenUsage": {
|
|
"inputTokens": 1000,
|
|
"outputTokens": 500,
|
|
"cachedInputTokens": 200
|
|
}
|
|
}
|
|
(session_dir / "generations" / "gen1").mkdir(parents=True)
|
|
(session_dir / "generations" / "gen1" / "projection-state.json").write_text(
|
|
json.dumps(projection_state)
|
|
)
|
|
|
|
# Import pricing module to get test data
|
|
from agent_cost_estimator import pricing as pricing_module
|
|
|
|
# Create minimal pricing data
|
|
pricing_data = {
|
|
"providers": {
|
|
"mistral": {
|
|
"currency": "USD",
|
|
"models": {
|
|
"default": {
|
|
"input_price": 1.5,
|
|
"cached_input_price": 0.15,
|
|
"output_price": 7.5
|
|
},
|
|
"mistral-medium": {
|
|
"input_price": 1.5,
|
|
"cached_input_price": 0.15,
|
|
"output_price": 7.5
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Create discovery and test
|
|
discovery = SessionDiscovery(pricing_data)
|
|
|
|
# Mock the VibeHarness to return our test data
|
|
with patch.object(VibeHarness, 'default_home', return_value=home):
|
|
# Test discovery
|
|
harnesses = discovery.discover_sessions()
|
|
|
|
# Check that vibe harness was found
|
|
assert "vibe" in harnesses
|
|
|
|
# Check that session was discovered
|
|
# (This may fail if the session structure isn't complete enough)
|
|
vibe_sessions = harnesses["vibe"].sessions
|
|
print(f"Discovered {len(vibe_sessions)} Vibe sessions")
|
|
# At least test that it doesn't crash
|
|
|
|
|
|
class TestHermesSessionDiscovery:
|
|
"""Test Hermes session discovery."""
|
|
|
|
def test_hermes_database_session_discovery(self):
|
|
"""Test discovering Hermes sessions from SQLite database."""
|
|
# Create a temporary SQLite database with test sessions
|
|
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmpdb:
|
|
db_path = pathlib.Path(tmpdb.name)
|
|
|
|
try:
|
|
# Create database and sessions table
|
|
conn = sqlite3.connect(str(db_path))
|
|
cursor = conn.cursor()
|
|
|
|
# Create sessions table (matching Hermes schema)
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS sessions (
|
|
id TEXT PRIMARY KEY,
|
|
title TEXT,
|
|
model TEXT,
|
|
started_at REAL,
|
|
last_activity_at REAL,
|
|
completed_at REAL
|
|
)
|
|
""")
|
|
|
|
# Insert test sessions
|
|
import time
|
|
now = time.time()
|
|
cursor.execute(
|
|
"INSERT INTO sessions (id, title, model, started_at, last_activity_at) "
|
|
"VALUES (?, ?, ?, ?, ?)",
|
|
("session-001", "Test Session 1", "mistral-medium", now - 3600, now)
|
|
)
|
|
cursor.execute(
|
|
"INSERT INTO sessions (id, title, model, started_at, last_activity_at) "
|
|
"VALUES (?, ?, ?, ?, ?)",
|
|
("session-002", "Test Session 2", "zai-glm-5-3", now - 7200, now - 100)
|
|
)
|
|
|
|
# Create session_model_usage table
|
|
cursor.execute("""
|
|
CREATE TABLE IF NOT EXISTS session_model_usage (
|
|
session_id TEXT,
|
|
model TEXT,
|
|
input_tokens INTEGER DEFAULT 0,
|
|
cached_input_tokens INTEGER DEFAULT 0,
|
|
output_tokens INTEGER DEFAULT 0,
|
|
PRIMARY KEY (session_id, model),
|
|
FOREIGN KEY (session_id) REFERENCES sessions(id)
|
|
)
|
|
""")
|
|
|
|
# Insert usage data
|
|
cursor.execute(
|
|
"INSERT INTO session_model_usage (session_id, model, input_tokens, cached_input_tokens, output_tokens) "
|
|
"VALUES (?, ?, ?, ?, ?)",
|
|
("session-001", "mistral-medium", 1000, 200, 500)
|
|
)
|
|
cursor.execute(
|
|
"INSERT INTO session_model_usage (session_id, model, input_tokens, cached_input_tokens, output_tokens) "
|
|
"VALUES (?, ?, ?, ?, ?)",
|
|
("session-002", "zai-glm-5-3", 2000, 400, 1000)
|
|
)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
# Create minimal pricing data
|
|
pricing_data = {
|
|
"providers": {
|
|
"mistral": {
|
|
"currency": "USD",
|
|
"models": {
|
|
"default": {
|
|
"input_price": 1.5,
|
|
"cached_input_price": 0.15,
|
|
"output_price": 7.5
|
|
},
|
|
"mistral-medium": {
|
|
"input_price": 1.5,
|
|
"cached_input_price": 0.15,
|
|
"output_price": 7.5
|
|
}
|
|
}
|
|
},
|
|
"zai": {
|
|
"currency": "USD",
|
|
"models": {
|
|
"zai-glm-5-3": {
|
|
"input_price": 1.5,
|
|
"cached_input_price": 0.15,
|
|
"output_price": 7.5
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Create discovery and test Hermes
|
|
discovery = SessionDiscovery(pricing_data)
|
|
|
|
# Create a temporary home directory
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
home = pathlib.Path(tmpdir)
|
|
|
|
# Copy the database to the home directory as state.db
|
|
target_db = home / "state.db"
|
|
import shutil
|
|
shutil.copy(db_path, target_db)
|
|
|
|
# Mock the HermesHarness default_home
|
|
with patch.object(HermesHarness, 'default_home', return_value=home):
|
|
# Mock the read_meta and collect_usage methods
|
|
with patch.object(HermesHarness, 'read_meta') as mock_read_meta:
|
|
def mock_read_meta_impl(session_dir, home=None):
|
|
# Extract session_id from path
|
|
if isinstance(session_dir, pathlib.Path):
|
|
session_id = session_dir.name
|
|
else:
|
|
from agent_cost_estimator.harnesses.hermes import HermesSession
|
|
session_id = session_dir.session_id
|
|
|
|
# Return mock metadata based on session_id
|
|
if session_id == "session-001":
|
|
from agent_cost_estimator.harnesses.base import SessionMeta
|
|
return SessionMeta(
|
|
title="Test Session 1",
|
|
model_name="mistral-medium",
|
|
alias=None
|
|
)
|
|
elif session_id == "session-002":
|
|
from agent_cost_estimator.harnesses.base import SessionMeta
|
|
return SessionMeta(
|
|
title="Test Session 2",
|
|
model_name="zai-glm-5-3",
|
|
alias=None
|
|
)
|
|
return SessionMeta(title=None, model_name=None, alias=None)
|
|
|
|
mock_read_meta.side_effect = mock_read_meta_impl
|
|
|
|
with patch.object(HermesHarness, 'collect_usage') as mock_collect_usage:
|
|
from agent_cost_estimator.harnesses.base import SessionUsage
|
|
|
|
def mock_collect_usage_impl(session_dir):
|
|
if isinstance(session_dir, pathlib.Path):
|
|
session_id = session_dir.name
|
|
else:
|
|
from agent_cost_estimator.harnesses.hermes import HermesSession
|
|
session_id = session_dir.session_id
|
|
|
|
if session_id == "session-001":
|
|
return SessionUsage(
|
|
input_tokens=1000,
|
|
cached_input_tokens=200,
|
|
output_tokens=500,
|
|
calls=10,
|
|
usage_source="state.db"
|
|
)
|
|
elif session_id == "session-002":
|
|
return SessionUsage(
|
|
input_tokens=2000,
|
|
cached_input_tokens=400,
|
|
output_tokens=1000,
|
|
calls=20,
|
|
usage_source="state.db"
|
|
)
|
|
return SessionUsage()
|
|
|
|
mock_collect_usage.side_effect = mock_collect_usage_impl
|
|
|
|
# Test discovery
|
|
harnesses = discovery.discover_sessions()
|
|
|
|
# Check that hermes harness was found
|
|
assert "hermes" in harnesses
|
|
|
|
# Check that sessions were discovered
|
|
hermes_sessions = harnesses["hermes"].sessions
|
|
print(f"Discovered {len(hermes_sessions)} Hermes sessions")
|
|
|
|
# Verify session details
|
|
if len(hermes_sessions) >= 1:
|
|
session = hermes_sessions[0]
|
|
assert session.harness_name == "hermes"
|
|
assert session.title in ["Test Session 1", "Test Session 2"]
|
|
assert session.model in ["mistral:mistral-medium", "zai:zai-glm-5-3"]
|
|
assert session.session_id in ["session-001", "session-002"]
|
|
|
|
finally:
|
|
# Clean up
|
|
db_path.unlink(missing_ok=True)
|
|
|
|
|
|
class TestHermesDiscoveryReal:
|
|
"""Test Hermes discovery with real adapter (if Hermes is installed)."""
|
|
|
|
def test_hermes_adapter_exists(self):
|
|
"""Test that HermesHarness can be instantiated."""
|
|
try:
|
|
from agent_cost_estimator.harnesses import HermesHarness
|
|
adapter = HermesHarness()
|
|
assert adapter.name == "hermes"
|
|
assert adapter.home_env_var == "HERMES_HOME"
|
|
assert adapter.home_dir_name == ".hermes"
|
|
except Exception as e:
|
|
pytest.skip(f"Hermes harness not available: {e}")
|
|
|
|
def test_hermes_home_detection(self):
|
|
"""Test Hermes home directory detection."""
|
|
try:
|
|
from agent_cost_estimator.harnesses import HermesHarness
|
|
adapter = HermesHarness()
|
|
home = adapter.default_home()
|
|
assert home.exists() or str(home).startswith("/home/")
|
|
except Exception as e:
|
|
pytest.skip(f"Cannot determine Hermes home: {e}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Run tests
|
|
import sys
|
|
sys.exit(pytest.main([__file__, "-v"]))
|