mirror of
https://github.com/norandom/OpenUltraSAST.git
synced 2026-10-16 03:49:26 +00:00
fix(model): the chat model is deepseek-flash; the hunter reaches DeepSeek
Every model call has failed since 2026-09-10 (83ab533):
- The default detector and judge model was `deepseek-v4.1-flash`, the product name.
The API rejects it with HTTP 400 ("The supported API model names are
deepseek-flash, deepseek-v4-pro"). The judge and pipeline record an endpoint
failure as a degradation, never a verdict, so nothing failed loudly.
- `tool_hunter.resolve_hunter_client` resolved OpenRouter only. In this project
DeepSeek is the chat provider and the OpenRouter key is scoped to embeddings, so
the hunter could not reach a chat model at all; the call came back 401.
The defaults are now `deepseek-flash` (priced). The hunter resolves DeepSeek first
when its key is set, and OpenRouter stays as the explicit or fallback route.
Verified live: plain completion and a tool call through both the endpoint resolver
and the hunter client.
Not affected: the independent evaluations (finding_dump runs with
max_model_calls=0) and the 2026-09-06 hunter measurement, which predates the id.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01EmM1j46qgKMh3zqD1QMwEo
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
5835239808
commit
cf579956d5
@@ -29,8 +29,12 @@ OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
|
||||
# 12:00 Beijing time on 2026-09-14, billed at Flash's price. So the judge tier is no longer a more expensive
|
||||
# model -- detector and judge are the same model, and any claim in this repo that the judge is a *stronger*
|
||||
# second opinion is now a claim about prompt and evidence, not about model class.
|
||||
DEFAULT_DETECTOR_MODEL = "deepseek-v4.1-flash"
|
||||
DEFAULT_JUDGE_MODEL = "deepseek-v4.1-flash"
|
||||
# The API name is `deepseek-flash`, not the product name. From 2026-09-10 to 2026-09-28 this default was
|
||||
# `deepseek-v4.1-flash`, which the platform rejects with HTTP 400 ("The supported API model names are
|
||||
# deepseek-flash, deepseek-v4-pro") -- every model call in that period failed and was recorded only as a
|
||||
# degradation. The id is what `GET /models` lists; `deepseek-chat` is an alias the platform maps to it.
|
||||
DEFAULT_DETECTOR_MODEL = "deepseek-flash"
|
||||
DEFAULT_JUDGE_MODEL = "deepseek-flash"
|
||||
Provider = Literal["deepseek", "openrouter", "custom", "scripted"]
|
||||
_SCRIPTED_FLAGS = frozenset({"scripted", "script", "dump", "dump-only", "unsafe", "unsafe-snippet"})
|
||||
|
||||
@@ -50,6 +54,7 @@ class Prices:
|
||||
|
||||
_PRICES: dict[str, Prices] = {
|
||||
# V4.1 Flash is priced as Flash; Pro requests are routed here and billed at this rate from 2026-09-14.
|
||||
"deepseek-flash": Prices(cache_hit_per_m=0.014, input_per_m=0.44, output_per_m=1.32),
|
||||
"deepseek-v4.1-flash": Prices(cache_hit_per_m=0.014, input_per_m=0.44, output_per_m=1.32),
|
||||
"deepseek-v4-flash": Prices(cache_hit_per_m=0.014, input_per_m=0.44, output_per_m=1.32),
|
||||
"deepseek-v4-flash-vision-exp": Prices(cache_hit_per_m=0.014, input_per_m=0.44, output_per_m=1.32),
|
||||
|
||||
@@ -237,7 +237,12 @@ def scripted_hunter_client(mode: str = "scripted") -> ScriptedChatClient:
|
||||
|
||||
|
||||
def resolve_hunter_client() -> ChatClient | None:
|
||||
"""Return a hunter client from env, or OpenRouter when an API key is present."""
|
||||
"""Return a hunter client: scripted when asked, DeepSeek when its key is set, else OpenRouter.
|
||||
|
||||
DeepSeek is the chat provider; OpenRouter carries embeddings. This resolved OpenRouter only, so with the
|
||||
project's keys the hunter could not reach the chat model at all -- every call went to an endpoint whose key
|
||||
is scoped to embeddings and came back 401.
|
||||
"""
|
||||
flag = os.environ.get(CLIENT_ENV, "").strip().lower()
|
||||
if flag in _SCRIPTED_FLAGS:
|
||||
return scripted_hunter_client("scripted")
|
||||
@@ -245,6 +250,12 @@ def resolve_hunter_client() -> ChatClient | None:
|
||||
return scripted_hunter_client("dump-only")
|
||||
if flag in _UNSAFE_FLAGS:
|
||||
return scripted_hunter_client("unsafe-snippet")
|
||||
if flag not in _OPENROUTER_FLAGS and os.environ.get("DEEPSEEK_API_KEY"):
|
||||
from .model.endpoint import DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL, ChatEndpoint, DeepSeekChatClient
|
||||
|
||||
base_url = os.environ.get(DEEPSEEK_BASE_ENV, DEEPSEEK_BASE_URL).rstrip("/")
|
||||
endpoint = ChatEndpoint(provider="deepseek", base_url=base_url, thinking=False)
|
||||
return DeepSeekChatClient(OpenRouterChatClient(api_key=os.environ["DEEPSEEK_API_KEY"], base_url=base_url), endpoint=endpoint)
|
||||
if flag in _OPENROUTER_FLAGS or (not flag and os.environ.get("OPENROUTER_API_KEY")):
|
||||
try:
|
||||
return OpenRouterHunterClient.from_env()
|
||||
|
||||
@@ -95,7 +95,7 @@ def test_the_config_can_point_the_chat_endpoint_somewhere_else(monkeypatch: pyte
|
||||
detector, judge = resolve_models(config)
|
||||
# The default detector follows DEFAULT_DETECTOR_MODEL; the judge here is pinned by config to a legacy id,
|
||||
# which must keep working so an old run can be reproduced and re-priced.
|
||||
assert detector == "deepseek-v4.1-flash" and judge == "deepseek-v4-pro"
|
||||
assert detector == "deepseek-flash" and judge == "deepseek-v4-pro"
|
||||
|
||||
|
||||
def test_the_adapter_disables_thinking_asks_for_json_and_keeps_the_reasoning_field() -> None:
|
||||
@@ -224,3 +224,25 @@ def test_the_config_can_turn_thinking_on(tmp_path: Path) -> None:
|
||||
assert load_config(path).models.thinking is True
|
||||
path.write_text("[models]\nthinking = false\n")
|
||||
assert load_config(path).models.thinking is False
|
||||
|
||||
|
||||
def test_the_default_model_is_an_api_name_the_platform_lists() -> None:
|
||||
"""`deepseek-v4.1-flash` is the product name; the API rejected it with HTTP 400 for every call (2026-09-10..28)."""
|
||||
from openultrasast.model.endpoint import DEFAULT_DETECTOR_MODEL, DEFAULT_JUDGE_MODEL, price_of
|
||||
|
||||
assert DEFAULT_DETECTOR_MODEL == DEFAULT_JUDGE_MODEL == "deepseek-flash"
|
||||
assert price_of(DEFAULT_DETECTOR_MODEL) is not None
|
||||
|
||||
|
||||
def test_the_hunter_reaches_deepseek_when_its_key_is_set(monkeypatch) -> None:
|
||||
"""DeepSeek is the chat provider; OpenRouter holds embeddings. The hunter resolved OpenRouter only."""
|
||||
from openultrasast import tool_hunter
|
||||
from openultrasast.model.endpoint import DeepSeekChatClient
|
||||
|
||||
monkeypatch.delenv(tool_hunter.CLIENT_ENV, raising=False)
|
||||
monkeypatch.setenv("DEEPSEEK_API_KEY", "test-key")
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "embeddings-key")
|
||||
client = tool_hunter.resolve_hunter_client()
|
||||
assert isinstance(client, DeepSeekChatClient), type(client)
|
||||
monkeypatch.delenv("DEEPSEEK_API_KEY")
|
||||
assert not isinstance(tool_hunter.resolve_hunter_client(), DeepSeekChatClient)
|
||||
|
||||
Reference in New Issue
Block a user