83 lines
4.2 KiB
Python
83 lines
4.2 KiB
Python
"""Optional local-model coaching based only on deterministic score findings.
|
||
|
||
The model may explain how to improve, but never changes metric values, points,
|
||
or pass/fail. Only failed criterion keys supplied by the scorer are accepted.
|
||
"""
|
||
|
||
import json
|
||
|
||
from app.config import get_settings
|
||
from app.dialog.llm import LlmClient, LlmRequest, LlmUnavailable, is_loopback_url
|
||
from app.domain.events import AICoaching, AIRecommendation, Metric
|
||
|
||
|
||
async def coach(metrics: list[Metric]) -> AICoaching:
|
||
failed = [item for item in metrics if not item.passed and item.weight > 0]
|
||
if not failed:
|
||
return AICoaching(status="not_needed")
|
||
|
||
settings = get_settings()
|
||
if not settings.assessment_feedback_enabled:
|
||
return AICoaching(status="disabled")
|
||
if (not settings.llm_model_control or not is_loopback_url(
|
||
settings.llm_control_base_url,
|
||
allow_docker_host=settings.allow_docker_host_models,
|
||
)):
|
||
return AICoaching(status="unavailable")
|
||
|
||
allowed = {item.key for item in failed}
|
||
schema = {"type": "json_object", "schema": {
|
||
"type": "object",
|
||
"properties": {"recommendations": {
|
||
"type": "array", "maxItems": 3,
|
||
"items": {"type": "object",
|
||
"properties": {
|
||
"metric_key": {"type": "string", "enum": sorted(allowed)},
|
||
"text": {"type": "string", "minLength": 12, "maxLength": 240},
|
||
},
|
||
"required": ["metric_key", "text"], "additionalProperties": False,
|
||
},
|
||
}},
|
||
"required": ["recommendations"], "additionalProperties": False,
|
||
}}
|
||
evidence = [{"metric_key": item.key, "criterion": item.title,
|
||
"observed": item.fact, "expected": item.norm}
|
||
for item in failed[:12]]
|
||
request = LlmRequest(
|
||
model=settings.llm_model_control,
|
||
messages=[{
|
||
"role": "system",
|
||
"content": (
|
||
"Ты методист учебного центра 112. По результатам детерминированной оценки "
|
||
"сформулируй до трёх коротких, конкретных рекомендаций курсанту: что "
|
||
"потренировать и как. Не пересчитывай баллы и не оспаривай зачёт. "
|
||
"Опирайся только на переданные наблюдения и нормативы; не придумывай "
|
||
"новые факты, требования и числа. Каждая рекомендация должна ссылаться "
|
||
"на один из переданных metric_key. Верни только JSON. /no_think"
|
||
),
|
||
}, {"role": "user", "content": json.dumps(evidence, ensure_ascii=False)}],
|
||
temperature=0.0, max_tokens=360, response_format=schema, strip_reasoning=True,
|
||
)
|
||
client = LlmClient(base_url=settings.llm_control_base_url, timeout=6)
|
||
try:
|
||
raw = await client.complete(request, use_cache=True)
|
||
payload = json.loads(raw)
|
||
if set(payload) != {"recommendations"} or not isinstance(payload["recommendations"], list):
|
||
raise ValueError("invalid coaching schema")
|
||
recommendations: list[AIRecommendation] = []
|
||
seen: set[str] = set()
|
||
for item in payload["recommendations"]:
|
||
if (not isinstance(item, dict) or set(item) != {"metric_key", "text"}
|
||
or item["metric_key"] not in allowed or item["metric_key"] in seen):
|
||
raise ValueError("recommendation references an unscored criterion")
|
||
recommendation = AIRecommendation.model_validate(item)
|
||
seen.add(recommendation.metric_key)
|
||
recommendations.append(recommendation)
|
||
if not recommendations:
|
||
raise ValueError("model returned no recommendations")
|
||
return AICoaching(status="ready", model=settings.llm_model_control,
|
||
recommendations=recommendations)
|
||
except (LlmUnavailable, ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||
return AICoaching(status="unavailable")
|
||
finally:
|
||
await client.aclose()
|