Fix LiteLLM cost model resolution

This commit is contained in:
Alex Schapiro
2026-08-12 17:26:00 +03:00
committed by Ahmed Allam
parent 7cc9fa9faa
commit 8ca0c4a9b8
5 changed files with 211 additions and 32 deletions
+1 -1
View File
@@ -143,7 +143,7 @@ def test_cost_callback_estimates_cost_with_bare_model_fallback() -> None:
}
def fake_completion_cost(**kwargs: object) -> float:
if kwargs["model"] == "gpt-4o-mini":
if kwargs["model"] == "openai/gpt-4o-mini":
return 0.025
raise ValueError(kwargs["model"])
+120
View File
@@ -0,0 +1,120 @@
from __future__ import annotations
from unittest.mock import patch
import litellm
from agents.usage import Usage
from strix.report.pricing import resolve_litellm_model
from strix.report.usage import LLMUsageLedger
def test_resolves_common_bare_model_names() -> None:
resolve_litellm_model.cache_clear()
assert resolve_litellm_model("deepseek-v4-flash") == "deepseek/deepseek-v4-flash"
assert resolve_litellm_model("openai/deepseek-v4-flash") == "deepseek/deepseek-v4-flash"
assert resolve_litellm_model("grok-4.5") == "xai/grok-4.5"
assert resolve_litellm_model("MiniMax-M3") == "minimax/MiniMax-M3"
def test_resolver_returns_none_for_unresolvable_model() -> None:
resolve_litellm_model.cache_clear()
assert resolve_litellm_model("provider/not-a-real-model") is None
def test_ledger_uses_estimate_when_routed_provider_reports_no_cost() -> None:
usage = Usage()
usage.requests = 1
usage.input_tokens = 1000
usage.output_tokens = 200
usage.total_tokens = 1200
ledger = LLMUsageLedger()
with patch("litellm.completion_cost", return_value=0.42):
ledger.record(agent_id="a", usage=usage, model="openai/deepseek-v4-flash")
assert ledger.total_cost == 0.42
def test_ledger_prefers_observed_cost_over_estimate() -> None:
usage = Usage()
usage.requests = 1
usage.input_tokens = 1000
usage.output_tokens = 200
usage.total_tokens = 1200
ledger = LLMUsageLedger()
with patch("litellm.completion_cost", return_value=0.42):
ledger.record(agent_id="a", usage=usage, model="openai/deepseek-v4-flash")
ledger.record_observed_cost(0.17)
assert ledger.total_cost == 0.17
def test_hydrated_estimate_continues_accumulating_new_estimates() -> None:
usage = Usage()
usage.requests = 1
usage.input_tokens = 1000
usage.output_tokens = 200
usage.total_tokens = 1200
ledger = LLMUsageLedger()
ledger.hydrate({"cost": 0.42})
with patch("litellm.completion_cost", return_value=0.17):
ledger.record(agent_id="a", usage=usage, model="openai/deepseek-v4-flash")
assert ledger.total_cost == 0.59
def test_zero_cost_disables_both_observed_and_estimated_costs() -> None:
usage = Usage()
usage.requests = 1
usage.input_tokens = 1000
usage.output_tokens = 200
usage.total_tokens = 1200
ledger = LLMUsageLedger()
ledger.zero_cost = True
with patch("litellm.completion_cost", return_value=0.42) as estimate:
ledger.record(agent_id="a", usage=usage, model="deepseek-v4-flash")
ledger.record_observed_cost(1.0)
estimate.assert_not_called()
assert ledger.total_cost == 0.0
def test_resolver_uses_provider_when_bare_entry_has_one() -> None:
original = litellm.model_cost
litellm.model_cost = {
"example": {
"litellm_provider": "example-provider",
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
}
}
try:
resolve_litellm_model.cache_clear()
assert resolve_litellm_model("example") == "example-provider/example"
finally:
litellm.model_cost = original
resolve_litellm_model.cache_clear()
def test_resolver_does_not_guess_between_differently_priced_providers() -> None:
original = litellm.model_cost
litellm.model_cost = {
"provider-a/example": {
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
},
"provider-b/example": {
"input_cost_per_token": 3.0,
"output_cost_per_token": 4.0,
},
}
try:
resolve_litellm_model.cache_clear()
assert resolve_litellm_model("example") is None
finally:
litellm.model_cost = original
resolve_litellm_model.cache_clear()