refactor: оценка занятия по неизменяемому снимку, грамматика параметром, тесты без SimpleNamespace
This commit is contained in:
gglamer
parent
a11206527a
commit
9f2ef7691f
5 changed files with 506 additions and 249 deletions
|
|
@ -29,7 +29,8 @@ from app.scoring.export import to_csv, to_pdf
|
|||
from app.scoring.report import build as build_report
|
||||
from app.session.access import can_access
|
||||
from app.session.checkpoint import load_state
|
||||
from app.session.finish import override_score, scoring_scenario
|
||||
from app.session.finish import override_score
|
||||
from app.session.score import scoring_scenario
|
||||
from app.session.hub import hub
|
||||
from app.session.store import ScoreOverridden, apply_score_override
|
||||
from app.voice.recording import recording_path
|
||||
|
|
|
|||
|
|
@ -12,22 +12,18 @@ import logging
|
|||
import time
|
||||
from uuid import UUID
|
||||
|
||||
from app.domain.events import CallEnded, CallEndReason, Exercise, Metric, ScoreReady, SessionEnded
|
||||
from app.domain.events import CallEnded, CallEndReason, Exercise, ScoreReady, SessionEnded
|
||||
from app.domain.statuses import ServiceStatus, current
|
||||
from app.domain.taxonomy import Competency, ErrorCode, Finding, FindingSource
|
||||
from app.domain.timers import TimerCode
|
||||
from app.scenarios import store
|
||||
from app.scoring.ai_coach import coach
|
||||
from app.scoring.card import evaluate_card
|
||||
from app.scoring.competency import radar
|
||||
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
||||
from app.scoring.gost import GostResult, evaluate
|
||||
from app.scoring.gost import GostResult
|
||||
from app.scoring.grammar import assess
|
||||
from app.scoring.report import build as build_report
|
||||
from app.scoring.taxonomy import METRIC_WEIGHTS
|
||||
from app.scoring.timing import time_credit
|
||||
from app.scoring.timing import DDS_WORK_TIME, time_metric
|
||||
from app.scoring.weights import apply_weights
|
||||
from app.session.hub import hub
|
||||
from app.session.score import ScoringSnapshot, score_session, scoring_scenario
|
||||
from app.session.state import DdsCardRecord, DdsLiveCard, now_utc
|
||||
from app.session.store import (
|
||||
LessonEnded,
|
||||
|
|
@ -77,39 +73,12 @@ def score_dds_card(state, card: DdsLiveCard) -> DdsCardRecord:
|
|||
}
|
||||
for service in services
|
||||
)
|
||||
work_passed = terminal and work_elapsed_ms is not None and work_elapsed_ms <= work_limit_ms
|
||||
work_delta_ms = (work_elapsed_ms - work_limit_ms) if work_elapsed_ms is not None else None
|
||||
if work_elapsed_ms is None:
|
||||
work_fact = "время обработки не зафиксировано"
|
||||
elif work_delta_ms and work_delta_ms > 0:
|
||||
work_fact = f"{round(work_elapsed_ms / 1000)} с (+{round(work_delta_ms / 1000)} с сверх норматива)"
|
||||
elif work_delta_ms and work_delta_ms < 0:
|
||||
work_fact = (f"{round(work_elapsed_ms / 1000)} с (на "
|
||||
f"{round(abs(work_delta_ms) / 1000)} с быстрее норматива)")
|
||||
else:
|
||||
work_fact = f"{round(work_elapsed_ms / 1000)} с (точно в норматив)"
|
||||
if not terminal:
|
||||
work_fact = f"карточка не завершена; {work_fact}"
|
||||
weighted.metrics.append(Metric(
|
||||
key="dds_work_time",
|
||||
title="Отработка карточки ДДС",
|
||||
fact=work_fact,
|
||||
norm=f"завершить за {round(work_limit_ms / 1000)} с",
|
||||
ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026",
|
||||
passed=work_passed,
|
||||
weight=METRIC_WEIGHTS["dds_work_time"],
|
||||
credit=time_credit(work_elapsed_ms, work_limit_ms) if terminal else 0.0,
|
||||
))
|
||||
if not work_passed:
|
||||
weighted.findings.append(Finding(
|
||||
code=ErrorCode.E3,
|
||||
source=FindingSource.TIMERS,
|
||||
summary="ДДС: норматив времени отработки карточки не выполнен",
|
||||
fact=work_fact,
|
||||
norm=f"завершить за {round(work_limit_ms / 1000)} с",
|
||||
ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026",
|
||||
competency=Competency.NORMS,
|
||||
))
|
||||
work_metric, work_finding = time_metric(
|
||||
DDS_WORK_TIME, work_elapsed_ms, work_limit_ms, completed=terminal
|
||||
)
|
||||
weighted.metrics.append(work_metric)
|
||||
if work_finding is not None:
|
||||
weighted.findings.append(work_finding)
|
||||
apply_weights(weighted, {**card.scenario.score_weights, **state.criteria.score_weights})
|
||||
actions = [
|
||||
{"type": "card.status", "service": mark.service, "status": mark.status.value,
|
||||
|
|
@ -149,20 +118,6 @@ def score_dds_card(state, card: DdsLiveCard) -> DdsCardRecord:
|
|||
)
|
||||
|
||||
|
||||
def scoring_scenario(state):
|
||||
"""Сценарий, по которому оценивается и разбирается занятие.
|
||||
|
||||
После передачи 112→ДДС КИО курсанта оценивается по эталону упражнения
|
||||
112. Балл, разбор и чек-лист самооценки обязаны брать один и тот же
|
||||
сценарий.
|
||||
"""
|
||||
if state.handoff_to_dds and state.operator_scenario:
|
||||
return state.operator_scenario
|
||||
if state.scenario is None and state.desk.scenarios:
|
||||
return state.desk.scenarios[0]
|
||||
return state.scenario or store.get(state.scenario_id)
|
||||
|
||||
|
||||
def score_live_dds_cards(state) -> list[DdsCardRecord]:
|
||||
"""Все карточки занятия: завершённые и оценка ещё открытых.
|
||||
|
||||
|
|
@ -182,185 +137,30 @@ async def finish(session_id: UUID, state) -> None:
|
|||
if state.recorder is not None:
|
||||
path = await asyncio.to_thread(state.recorder.finalize)
|
||||
state.recording_path = str(path) if path else None
|
||||
# Все карточки выданы одновременно: при досрочном завершении оцениваем
|
||||
# каждую, включая не открытую, потому что её норматив уже шёл.
|
||||
if state.exercise is Exercise.DDS or (state.handoff_to_dds and state.desk.scenarios):
|
||||
score_live_dds_cards(state)
|
||||
# Сценарий занятия, а не библиотечный: директивы могли поправить эталон.
|
||||
scenario = scoring_scenario(state)
|
||||
if scenario is None:
|
||||
snapshot = ScoringSnapshot.of(state)
|
||||
if snapshot is None:
|
||||
return
|
||||
|
||||
cards: list[DdsCardRecord] = []
|
||||
if state.exercise is Exercise.CARD:
|
||||
result = evaluate_card(
|
||||
scenario, state.operator_kio or state.dispatched_card or state.kio
|
||||
)
|
||||
limit_ms = state.timers.limits[TimerCode.CARD_FILL]
|
||||
elapsed_ms = state.timers.measured_ms(TimerCode.CARD_FILL)
|
||||
submitted = state.dispatched_card is not None
|
||||
if elapsed_ms is None:
|
||||
elapsed_fact = "время не зафиксировано"
|
||||
else:
|
||||
delta_ms = elapsed_ms - limit_ms
|
||||
elapsed_seconds = round(elapsed_ms / 1000)
|
||||
if delta_ms > 0:
|
||||
deviation = f"+{round(delta_ms / 1000)} с сверх норматива"
|
||||
elif delta_ms < 0:
|
||||
deviation = f"на {round(abs(delta_ms) / 1000)} с быстрее норматива"
|
||||
else:
|
||||
deviation = "точно в норматив"
|
||||
elapsed_fact = f"{elapsed_seconds} с ({deviation})"
|
||||
passed = submitted and elapsed_ms is not None and elapsed_ms <= limit_ms
|
||||
result.metrics.append(Metric(
|
||||
key="card_fill_time",
|
||||
title="Время заполнения карточки",
|
||||
fact=elapsed_fact if submitted else f"карточка не сдана; {elapsed_fact}",
|
||||
norm=f"сдать карточку за {round(limit_ms / 1000)} с",
|
||||
ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026",
|
||||
passed=passed,
|
||||
weight=METRIC_WEIGHTS["card_fill_time"],
|
||||
credit=time_credit(elapsed_ms, limit_ms) if submitted else 0.0,
|
||||
))
|
||||
if not passed:
|
||||
result.findings.append(Finding(
|
||||
code=ErrorCode.E3,
|
||||
source=FindingSource.TIMERS,
|
||||
summary="Время заполнения карточки: норматив не выполнен",
|
||||
fact=elapsed_fact if submitted else f"карточка не сдана; {elapsed_fact}",
|
||||
norm=f"сдать карточку за {round(limit_ms / 1000)} с",
|
||||
ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026",
|
||||
competency=Competency.NORMS,
|
||||
))
|
||||
elif state.exercise is Exercise.DDS:
|
||||
# Все карточки выданы одновременно: при досрочном завершении оцениваем
|
||||
# каждую, включая не открытую, потому что её норматив уже шёл.
|
||||
result = GostResult()
|
||||
cards = score_live_dds_cards(state)
|
||||
for card in cards:
|
||||
result.metrics.extend(card.metrics)
|
||||
result.findings.extend(card.findings)
|
||||
else:
|
||||
result = evaluate(
|
||||
scenario=scenario,
|
||||
kio=state.kio,
|
||||
timers=state.timers,
|
||||
revealed_facts=[fact.id for fact in state.slots.revealed_facts()] if state.slots else None,
|
||||
refined_facts=list(state.slots.refined) if state.slots else None,
|
||||
end_reason=state.end_reason,
|
||||
bounced_fields=state.bounced_fields,
|
||||
resolved_outcome=state.resolved_outcome,
|
||||
dispatched=state.dispatched_card is not None,
|
||||
)
|
||||
# Грамматика относится к свободному описанию оператора 112, а не к
|
||||
# заполнению карточки на стороне ДДС. Учитывается только по явной настройке
|
||||
# преподавателя и только когда есть текст для проверки.
|
||||
operator_kio = state.operator_kio if state.handoff_to_dds else None
|
||||
description = (operator_kio or state.kio).description or ""
|
||||
if (state.exercise is not Exercise.DDS and state.criteria.require_correct_grammar
|
||||
and description.strip()):
|
||||
grammar = await assess(description)
|
||||
result.metrics.append(Metric(
|
||||
key="description_grammar",
|
||||
title="Грамматика описания происшествия",
|
||||
fact="ошибок не обнаружено" if grammar.passed else "; ".join(grammar.errors),
|
||||
norm="грамматически корректное описание",
|
||||
ref="критерий занятия; правила русского языка",
|
||||
passed=grammar.passed,
|
||||
weight=1.0,
|
||||
))
|
||||
if not grammar.passed:
|
||||
result.findings.append(Finding(
|
||||
code=ErrorCode.E4,
|
||||
source=FindingSource.GRAMMAR,
|
||||
summary="Грамматическая ошибка в описании происшествия",
|
||||
fact="; ".join(grammar.errors),
|
||||
norm="грамматически корректное описание",
|
||||
ref="критерий занятия; правила русского языка",
|
||||
competency=Competency.COMMUNICATION,
|
||||
))
|
||||
# Работа диспетчера — вторая роль и вторая таксономия. Отметки D1–D6 идут
|
||||
# рядом с E1–E6, а не вместо: в живой цепочке 112 → ДДС в одном занятии
|
||||
# участвуют оба (docs/spec/DATASET.md#статусы-реагирования).
|
||||
live_card = state.desk.active
|
||||
if live_card is not None and state.exercise is Exercise.CALL:
|
||||
decision_limit_ms = live_card.timers.limits[TimerCode.DDS_ACK]
|
||||
services = state.card_services(live_card)
|
||||
dispatcher_findings = evaluate_dispatcher(
|
||||
entries=live_card.status_log,
|
||||
services=services,
|
||||
crew_assignments=live_card.crew_assignments,
|
||||
deadline_ms=decision_limit_ms,
|
||||
elapsed_ms=live_card.timers.measured_ms(TimerCode.DDS_ACK),
|
||||
reply_text=live_card.reply_text,
|
||||
expected_decision=scenario.dds_decision.expected,
|
||||
expected_decision_reason=scenario.dds_decision.reason,
|
||||
)
|
||||
result.findings.extend(dispatcher_findings)
|
||||
result.metrics.extend(dispatcher_metrics(
|
||||
live_card, decision_limit_ms, scenario.dds_decision.expected,
|
||||
scenario.dds_decision.reason, services=services,
|
||||
))
|
||||
# DDS cards were weighted individually in score_dds_card using each
|
||||
# card's scenario defaults plus the lesson override. Reapplying the first
|
||||
# scenario's weights here would corrupt the other ticket cards.
|
||||
if state.exercise is not Exercise.DDS:
|
||||
apply_weights(result, {**scenario.score_weights, **state.criteria.score_weights})
|
||||
|
||||
if state.handoff_to_dds and state.desk.scenarios:
|
||||
# В связке КИО оценивается относительно эталона и весов упражнения
|
||||
# 112, а каждая карточка очереди уже взвешена собственным сценарием.
|
||||
cards = score_live_dds_cards(state)
|
||||
for card in cards:
|
||||
result.metrics.extend(card.metrics)
|
||||
result.findings.extend(card.findings)
|
||||
|
||||
# Сводка числами: по ней считается дельта между попытками в профиле.
|
||||
# Вытаскивать её разбором текста метрик («94 с») — путь к тихим ошибкам.
|
||||
required = scenario.ground_truth.required_facts
|
||||
revealed = [fact.id for fact in state.slots.revealed_facts()] if state.slots else []
|
||||
codes: dict[str, int] = {}
|
||||
for finding in result.findings:
|
||||
codes[finding.code.value] = codes.get(finding.code.value, 0) + 1
|
||||
|
||||
state.score = {
|
||||
"score_auto": result.score,
|
||||
"summary": {
|
||||
"interview_ms": state.timers.measured_ms(TimerCode.INTERVIEW),
|
||||
"card_fill_ms": state.timers.measured_ms(TimerCode.CARD_FILL),
|
||||
"facts_got": len([fact for fact in required if fact in revealed]),
|
||||
"facts_required": len(required),
|
||||
"hints": len(state.hints_shown),
|
||||
"codes": codes,
|
||||
},
|
||||
"metrics": [metric.model_dump() for metric in result.metrics],
|
||||
"findings": [finding.model_dump(mode="json") for finding in result.findings],
|
||||
"competencies": [item.model_dump() for item in radar(result.metrics)],
|
||||
"unavailable": result.unavailable,
|
||||
# Статус карточки — готовая красная метка, понятная любому диспетчеру.
|
||||
"card_status": state.station_snapshot().card.value,
|
||||
"card_results": [
|
||||
{"card_id": str(card.card_id), "scenario_id": card.scenario_id,
|
||||
"score_auto": card.score_auto, "reply_text": card.reply_text,
|
||||
"actions": card.actions, "duration_ms": card.duration_ms,
|
||||
"title": card.title, "address": card.address,
|
||||
"description": card.description, "incident_type": card.incident_type,
|
||||
"victims_count": card.victims_count,
|
||||
"received_at": card.received_at.isoformat() if card.received_at else None,
|
||||
"managed_service": card.managed_service,
|
||||
"recipient_services": card.recipient_services,
|
||||
"metrics": [metric.model_dump(mode="json") for metric in card.metrics],
|
||||
"findings": [finding.model_dump(mode="json") for finding in card.findings]}
|
||||
for card in cards
|
||||
] if state.exercise is Exercise.DDS or state.handoff_to_dds else [],
|
||||
}
|
||||
report = await score_session(snapshot, grammar=assess)
|
||||
state.score = report.payload(card_status=state.station_snapshot().card.value)
|
||||
# Модель только поясняет уже посчитанные провалы и балл не трогает; без
|
||||
# запущенной модели разбор выходит со статусом «недоступно», а не ждёт её.
|
||||
state.score["ai_coaching"] = (await coach(result.metrics)).model_dump(mode="json")
|
||||
state.score["ai_coaching"] = (await coach(report.result.metrics)).model_dump(mode="json")
|
||||
# Полный разбор хранится вместе с оценкой: PDF/CSV и история должны
|
||||
# переживать перезапуск backend, а не зависеть от живого объекта в реестре хаба.
|
||||
state.score["full_report"] = build_report(session_id, state, scenario).model_dump(mode="json")
|
||||
log.info("сессия %s: оценка %.1f, отметок %d", session_id, result.score, len(result.findings))
|
||||
state.score["full_report"] = build_report(
|
||||
session_id, state, snapshot.scenario
|
||||
).model_dump(mode="json")
|
||||
log.info("сессия %s: оценка %.1f, отметок %d",
|
||||
session_id, report.result.score, len(report.result.findings))
|
||||
|
||||
# Оценка и аудит уходят в commit операции, завершившей занятие; ScoreReady
|
||||
# рассылает `end_session` и ждёт того же commit — без записи итог не выдаётся.
|
||||
hub.record(session_id, ScoreCalculated(result.score, state.score))
|
||||
hub.record(session_id, ScoreCalculated(report.result.score, state.score))
|
||||
|
||||
|
||||
def _ends_on_station(state) -> bool:
|
||||
|
|
|
|||
263
backend/app/session/score.py
Normal file
263
backend/app/session/score.py
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
"""Оценка занятия по неизменяемому снимку.
|
||||
|
||||
`finish()` сначала доводит живое занятие до оцениваемого вида (дооценивает
|
||||
очередь ДДС), затем снимает `ScoringSnapshot` и отдаёт его `score_session`.
|
||||
Сам расчёт не читает и не меняет `state`: живой, сохранённый и перестроенный
|
||||
отчёт получают сценарий одним путём — через `scoring_scenario`.
|
||||
"""
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from app.domain.events import CallEndReason, Exercise, LessonCriteria, Metric
|
||||
from app.domain.kio import KIO
|
||||
from app.domain.taxonomy import Competency, ErrorCode, Finding, FindingSource
|
||||
from app.domain.timers import TimerCode
|
||||
from app.scenarios import store
|
||||
from app.scenarios.schema import Scenario
|
||||
from app.scoring.card import evaluate_card
|
||||
from app.scoring.competency import radar
|
||||
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
||||
from app.scoring.gost import GostResult, evaluate
|
||||
from app.scoring.grammar import GrammarAssessment, assess
|
||||
from app.scoring.timing import CARD_FILL_TIME, time_metric
|
||||
from app.scoring.weights import apply_weights
|
||||
from app.session.dds import DdsCardRecord, DdsLiveCard
|
||||
from app.session.timers import SessionTimers
|
||||
|
||||
GrammarCheck = Callable[[str], Awaitable[GrammarAssessment]]
|
||||
|
||||
|
||||
def scoring_scenario(state) -> Scenario | None:
|
||||
"""Сценарий, по которому оценивается и разбирается занятие.
|
||||
|
||||
После передачи 112→ДДС КИО курсанта оценивается по эталону упражнения
|
||||
112. Балл, разбор и чек-лист самооценки обязаны брать один и тот же
|
||||
сценарий.
|
||||
"""
|
||||
if state.handoff_to_dds and state.operator_scenario:
|
||||
return state.operator_scenario
|
||||
if state.scenario is None and state.desk.scenarios:
|
||||
return state.desk.scenarios[0]
|
||||
return state.scenario or store.get(state.scenario_id)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScoringSnapshot:
|
||||
"""Всё, что нужно для балла, снятое с занятия в момент завершения."""
|
||||
|
||||
scenario: Scenario
|
||||
exercise: Exercise
|
||||
handoff_to_dds: bool
|
||||
#: КИО, которую сверяют с эталоном: в упражнении 112 — сданная карточка.
|
||||
kio: KIO
|
||||
#: Свободное описание оператора 112 — предмет проверки грамматики.
|
||||
description: str
|
||||
dispatched: bool
|
||||
timers: SessionTimers
|
||||
criteria: LessonCriteria
|
||||
revealed_facts: tuple[str, ...] | None
|
||||
refined_facts: tuple[str, ...] | None
|
||||
end_reason: CallEndReason | None
|
||||
bounced_fields: tuple[str, ...]
|
||||
resolved_outcome: str | None
|
||||
hints: int
|
||||
#: Уже оценённые карточки очереди ДДС, в порядке выдачи.
|
||||
dds_cards: tuple[DdsCardRecord, ...]
|
||||
#: Карточка на пульте в упражнении по звонку: отметки D1–D6 рядом с E1–E6.
|
||||
call_card: DdsLiveCard | None
|
||||
call_card_services: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def reports_cards(self) -> bool:
|
||||
return self.exercise is Exercise.DDS or self.handoff_to_dds
|
||||
|
||||
@classmethod
|
||||
def of(cls, state) -> "ScoringSnapshot | None":
|
||||
"""Снимок живого занятия; None — оценивать не по чему.
|
||||
|
||||
Очередь ДДС к этому моменту должна быть дооценена
|
||||
(`finish.score_live_dds_cards`): снимок берёт готовые записи.
|
||||
"""
|
||||
scenario = scoring_scenario(state)
|
||||
if scenario is None:
|
||||
return None
|
||||
if state.exercise is Exercise.CARD:
|
||||
kio = state.operator_kio or state.dispatched_card or state.kio
|
||||
else:
|
||||
kio = state.kio
|
||||
# Грамматика относится к свободному описанию оператора 112, а не к
|
||||
# заполнению карточки на стороне ДДС.
|
||||
operator_kio = state.operator_kio if state.handoff_to_dds else None
|
||||
live_card = state.desk.active if state.exercise is Exercise.CALL else None
|
||||
queue = state.exercise is Exercise.DDS or (state.handoff_to_dds and state.desk.scenarios)
|
||||
return cls(
|
||||
scenario=scenario,
|
||||
exercise=state.exercise,
|
||||
handoff_to_dds=state.handoff_to_dds,
|
||||
kio=kio.model_copy(deep=True),
|
||||
description=(operator_kio or state.kio).description or "",
|
||||
dispatched=state.dispatched_card is not None,
|
||||
timers=state.timers.model_copy(deep=True),
|
||||
criteria=state.criteria.model_copy(deep=True),
|
||||
revealed_facts=(tuple(fact.id for fact in state.slots.revealed_facts())
|
||||
if state.slots else None),
|
||||
refined_facts=tuple(state.slots.refined) if state.slots else None,
|
||||
end_reason=state.end_reason,
|
||||
bounced_fields=tuple(state.bounced_fields),
|
||||
resolved_outcome=state.resolved_outcome,
|
||||
hints=len(state.hints_shown),
|
||||
dds_cards=tuple(state.desk.completed) if queue else (),
|
||||
call_card=live_card.model_copy(deep=True) if live_card is not None else None,
|
||||
call_card_services=(tuple(state.card_services(live_card))
|
||||
if live_card is not None else ()),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScoreReport:
|
||||
"""Посчитанная оценка: результат ГОСТ, сводка числами и карточки ДДС."""
|
||||
|
||||
result: GostResult
|
||||
summary: dict[str, Any]
|
||||
cards: tuple[DdsCardRecord, ...]
|
||||
reports_cards: bool
|
||||
|
||||
def payload(self, *, card_status: str) -> dict[str, Any]:
|
||||
"""Сохраняемая оценка занятия — без разбора модели и полного отчёта."""
|
||||
return {
|
||||
"score_auto": self.result.score,
|
||||
"summary": self.summary,
|
||||
"metrics": [metric.model_dump() for metric in self.result.metrics],
|
||||
"findings": [finding.model_dump(mode="json") for finding in self.result.findings],
|
||||
"competencies": [item.model_dump() for item in radar(self.result.metrics)],
|
||||
"unavailable": self.result.unavailable,
|
||||
# Статус карточки — готовая красная метка, понятная любому диспетчеру.
|
||||
"card_status": card_status,
|
||||
"card_results": [
|
||||
{"card_id": str(card.card_id), "scenario_id": card.scenario_id,
|
||||
"score_auto": card.score_auto, "reply_text": card.reply_text,
|
||||
"actions": card.actions, "duration_ms": card.duration_ms,
|
||||
"title": card.title, "address": card.address,
|
||||
"description": card.description, "incident_type": card.incident_type,
|
||||
"victims_count": card.victims_count,
|
||||
"received_at": card.received_at.isoformat() if card.received_at else None,
|
||||
"managed_service": card.managed_service,
|
||||
"recipient_services": card.recipient_services,
|
||||
"metrics": [metric.model_dump(mode="json") for metric in card.metrics],
|
||||
"findings": [finding.model_dump(mode="json") for finding in card.findings]}
|
||||
for card in self.cards
|
||||
] if self.reports_cards else [],
|
||||
}
|
||||
|
||||
|
||||
async def score_session(
|
||||
snapshot: ScoringSnapshot, *, grammar: GrammarCheck = assess
|
||||
) -> ScoreReport:
|
||||
"""Балл занятия. Единственный внешний вызов — проверка грамматики, и та
|
||||
только по явному критерию преподавателя."""
|
||||
scenario = snapshot.scenario
|
||||
if snapshot.exercise is Exercise.CARD:
|
||||
result = evaluate_card(scenario, snapshot.kio)
|
||||
metric, finding = time_metric(
|
||||
CARD_FILL_TIME,
|
||||
snapshot.timers.measured_ms(TimerCode.CARD_FILL),
|
||||
snapshot.timers.limits[TimerCode.CARD_FILL],
|
||||
completed=snapshot.dispatched,
|
||||
)
|
||||
result.metrics.append(metric)
|
||||
if finding is not None:
|
||||
result.findings.append(finding)
|
||||
elif snapshot.exercise is Exercise.DDS:
|
||||
# Все карточки выданы одновременно: при досрочном завершении оценена
|
||||
# каждая, включая не открытую, потому что её норматив уже шёл.
|
||||
result = GostResult()
|
||||
else:
|
||||
result = evaluate(
|
||||
scenario=scenario,
|
||||
kio=snapshot.kio,
|
||||
timers=snapshot.timers,
|
||||
revealed_facts=(list(snapshot.revealed_facts)
|
||||
if snapshot.revealed_facts is not None else None),
|
||||
refined_facts=(list(snapshot.refined_facts)
|
||||
if snapshot.refined_facts is not None else None),
|
||||
end_reason=snapshot.end_reason,
|
||||
bounced_fields=list(snapshot.bounced_fields),
|
||||
resolved_outcome=snapshot.resolved_outcome,
|
||||
dispatched=snapshot.dispatched,
|
||||
)
|
||||
# Учитывается только по явной настройке преподавателя и только когда есть
|
||||
# текст для проверки.
|
||||
if (snapshot.exercise is not Exercise.DDS and snapshot.criteria.require_correct_grammar
|
||||
and snapshot.description.strip()):
|
||||
verdict = await grammar(snapshot.description)
|
||||
result.metrics.append(Metric(
|
||||
key="description_grammar",
|
||||
title="Грамматика описания происшествия",
|
||||
fact="ошибок не обнаружено" if verdict.passed else "; ".join(verdict.errors),
|
||||
norm="грамматически корректное описание",
|
||||
ref="критерий занятия; правила русского языка",
|
||||
passed=verdict.passed,
|
||||
weight=1.0,
|
||||
))
|
||||
if not verdict.passed:
|
||||
result.findings.append(Finding(
|
||||
code=ErrorCode.E4,
|
||||
source=FindingSource.GRAMMAR,
|
||||
summary="Грамматическая ошибка в описании происшествия",
|
||||
fact="; ".join(verdict.errors),
|
||||
norm="грамматически корректное описание",
|
||||
ref="критерий занятия; правила русского языка",
|
||||
competency=Competency.COMMUNICATION,
|
||||
))
|
||||
# Работа диспетчера — вторая роль и вторая таксономия. Отметки D1–D6 идут
|
||||
# рядом с E1–E6, а не вместо: в живой цепочке 112 → ДДС в одном занятии
|
||||
# участвуют оба (docs/spec/DATASET.md#статусы-реагирования).
|
||||
live_card = snapshot.call_card
|
||||
if live_card is not None:
|
||||
decision_limit_ms = live_card.timers.limits[TimerCode.DDS_ACK]
|
||||
services = list(snapshot.call_card_services)
|
||||
result.findings.extend(evaluate_dispatcher(
|
||||
entries=live_card.status_log,
|
||||
services=services,
|
||||
crew_assignments=live_card.crew_assignments,
|
||||
deadline_ms=decision_limit_ms,
|
||||
elapsed_ms=live_card.timers.measured_ms(TimerCode.DDS_ACK),
|
||||
reply_text=live_card.reply_text,
|
||||
expected_decision=scenario.dds_decision.expected,
|
||||
expected_decision_reason=scenario.dds_decision.reason,
|
||||
))
|
||||
result.metrics.extend(dispatcher_metrics(
|
||||
live_card, decision_limit_ms, scenario.dds_decision.expected,
|
||||
scenario.dds_decision.reason, services=services,
|
||||
))
|
||||
# Карточки ДДС уже взвешены каждая своим сценарием плюс настройкой
|
||||
# занятия. Веса первого сценария поверх них испортили бы остальные.
|
||||
if snapshot.exercise is not Exercise.DDS:
|
||||
apply_weights(result, {**scenario.score_weights, **snapshot.criteria.score_weights})
|
||||
# В связке КИО оценивается относительно эталона и весов упражнения 112,
|
||||
# а каждая карточка очереди уже взвешена собственным сценарием.
|
||||
for card in snapshot.dds_cards:
|
||||
result.metrics.extend(card.metrics)
|
||||
result.findings.extend(card.findings)
|
||||
|
||||
# Сводка числами: по ней считается дельта между попытками в профиле.
|
||||
# Вытаскивать её разбором текста метрик («94 с») — путь к тихим ошибкам.
|
||||
required = scenario.ground_truth.required_facts
|
||||
revealed = snapshot.revealed_facts or ()
|
||||
codes: dict[str, int] = {}
|
||||
for finding in result.findings:
|
||||
codes[finding.code.value] = codes.get(finding.code.value, 0) + 1
|
||||
summary = {
|
||||
"interview_ms": snapshot.timers.measured_ms(TimerCode.INTERVIEW),
|
||||
"card_fill_ms": snapshot.timers.measured_ms(TimerCode.CARD_FILL),
|
||||
"facts_got": len([fact for fact in required if fact in revealed]),
|
||||
"facts_required": len(required),
|
||||
"hints": snapshot.hints,
|
||||
"codes": codes,
|
||||
}
|
||||
return ScoreReport(
|
||||
result=result, summary=summary, cards=snapshot.dds_cards,
|
||||
reports_cards=snapshot.reports_cards,
|
||||
)
|
||||
|
|
@ -1,12 +1,28 @@
|
|||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.domain.statuses import ServiceStatus, StatusEntry
|
||||
from app.domain.taxonomy import ErrorCode
|
||||
from app.scenarios.loader import load_file
|
||||
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
||||
from app.session.dds import DdsLiveCard, build_card
|
||||
from app.session.timers import SessionTimers
|
||||
|
||||
LIBRARY = Path(__file__).resolve().parents[2] / "scenarios"
|
||||
SERVICES = ["Служба 101"]
|
||||
|
||||
|
||||
def live_card(*, status_log, crew_assignments, dispatched_at) -> DdsLiveCard:
|
||||
"""Карточка пульта с заданным журналом статусов — вход оценки ДДС."""
|
||||
scenario = load_file(LIBRARY / "fire-apartment-l2.yaml", LIBRARY)
|
||||
card = build_card(scenario, 0, SessionTimers().limits)
|
||||
card.status_log = status_log
|
||||
card.crew_assignments = crew_assignments
|
||||
card.dispatched_at = dispatched_at
|
||||
return card
|
||||
|
||||
|
||||
def test_late_primary_status_has_d1_finding_even_when_status_exists():
|
||||
|
|
@ -110,8 +126,7 @@ def test_missing_manual_status_comment_has_a_d5_finding():
|
|||
|
||||
def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
||||
at = datetime.now(timezone.utc)
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"],
|
||||
card = live_card(
|
||||
status_log=[
|
||||
StatusEntry(
|
||||
service="Служба 101", status=ServiceStatus.ACCEPTED,
|
||||
|
|
@ -127,7 +142,7 @@ def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
|||
)
|
||||
|
||||
reply = next(
|
||||
metric for metric in dispatcher_metrics(state, 30_000)
|
||||
metric for metric in dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||
if metric.key == "dds_reply"
|
||||
)
|
||||
|
||||
|
|
@ -135,9 +150,9 @@ def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
|||
assert "к каждой ручной отметке" in reply.norm
|
||||
|
||||
findings = evaluate_dispatcher(
|
||||
entries=state.status_log,
|
||||
entries=card.status_log,
|
||||
services=["Служба 101"],
|
||||
crew_assignments=state.crew_assignments,
|
||||
crew_assignments=card.crew_assignments,
|
||||
deadline_ms=30_000,
|
||||
elapsed_ms=0,
|
||||
)
|
||||
|
|
@ -173,13 +188,13 @@ def test_scenario_may_define_a_valid_decline_for_duplicate_or_territory():
|
|||
expected_decision_reason="дублирующая карточка",
|
||||
)
|
||||
assert ErrorCode.D3 not in [finding.code for finding in findings]
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
||||
card = live_card(
|
||||
status_log=entries,
|
||||
crew_assignments={}, dispatched_at=at,
|
||||
)
|
||||
metrics = dispatcher_metrics(
|
||||
state, 30_000, expected_decision="decline",
|
||||
expected_decision_reason="дублирующая карточка",
|
||||
card, 30_000, expected_decision="decline",
|
||||
expected_decision_reason="дублирующая карточка", services=SERVICES,
|
||||
)
|
||||
decision = next(metric for metric in metrics if metric.key == "dds_decision")
|
||||
assert decision.passed
|
||||
|
|
@ -188,8 +203,7 @@ def test_scenario_may_define_a_valid_decline_for_duplicate_or_territory():
|
|||
|
||||
def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes():
|
||||
at = datetime.now(timezone.utc)
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"],
|
||||
card = live_card(
|
||||
status_log=[StatusEntry(
|
||||
service="Служба 101", status=ServiceStatus.ACCEPTED,
|
||||
at=at, comment="бригада на связи",
|
||||
|
|
@ -197,19 +211,19 @@ def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes(
|
|||
crew_assignments={"Служба 101": "Бригада 12"},
|
||||
dispatched_at=at,
|
||||
)
|
||||
reply = next(m for m in dispatcher_metrics(state, 30_000) if m.key == "dds_reply")
|
||||
reply = next(m for m in dispatcher_metrics(card, 30_000, services=SERVICES) if m.key == "dds_reply")
|
||||
assert not reply.passed
|
||||
findings = evaluate_dispatcher(
|
||||
entries=state.status_log, services=["Служба 101"],
|
||||
crew_assignments=state.crew_assignments,
|
||||
entries=card.status_log, services=["Служба 101"],
|
||||
crew_assignments=card.crew_assignments,
|
||||
deadline_ms=30_000, elapsed_ms=0,
|
||||
)
|
||||
assert any("не разделяют основание и сведения" in f.summary for f in findings)
|
||||
|
||||
state.status_log[0] = state.status_log[0].model_copy(update={
|
||||
card.status_log[0] = card.status_log[0].model_copy(update={
|
||||
"comment": "Основание: доклад старшего.\nСведения: бригада на связи.",
|
||||
})
|
||||
reply = next(m for m in dispatcher_metrics(state, 30_000) if m.key == "dds_reply")
|
||||
reply = next(m for m in dispatcher_metrics(card, 30_000, services=SERVICES) if m.key == "dds_reply")
|
||||
assert reply.passed
|
||||
|
||||
def test_accepting_card_with_scenario_expected_decline_is_explained():
|
||||
|
|
@ -242,11 +256,11 @@ def test_incomplete_work_path_has_d6_for_failed_progress_metrics():
|
|||
StatusEntry(service="Служба 101", status=ServiceStatus.ACCEPTED, at=at),
|
||||
StatusEntry(service="Служба 101", status=ServiceStatus.RESPONDING, at=at),
|
||||
]
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
||||
card = live_card(
|
||||
status_log=entries,
|
||||
crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at,
|
||||
)
|
||||
metrics = dispatcher_metrics(state, 30_000)
|
||||
metrics = dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||
assert not next(item for item in metrics if item.key == "dds_progress").passed
|
||||
assert not next(item for item in metrics if item.key == "dds_completion").passed
|
||||
findings = evaluate_dispatcher(
|
||||
|
|
@ -267,11 +281,11 @@ def test_reasoned_refusal_after_acceptance_is_a_valid_terminal_path():
|
|||
comment="Бригаде переданы сведения, выезд не выполнялся по причине угрозы.",
|
||||
),
|
||||
]
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
||||
card = live_card(
|
||||
status_log=entries,
|
||||
crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at,
|
||||
)
|
||||
metrics = dispatcher_metrics(state, 30_000)
|
||||
metrics = dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||
assert next(item for item in metrics if item.key == "dds_progress").passed
|
||||
assert next(item for item in metrics if item.key == "dds_completion").passed
|
||||
findings = evaluate_dispatcher(
|
||||
|
|
|
|||
179
backend/tests/test_score_session.py
Normal file
179
backend/tests/test_score_session.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
"""Оценка занятия по снимку: без websocket, без живого `state` внутри расчёта."""
|
||||
|
||||
import asyncio
|
||||
from dataclasses import FrozenInstanceError
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.domain.events import Exercise, LessonCriteria, SessionMode
|
||||
from app.domain.kio import KIO
|
||||
from app.domain.statuses import ServiceStatus
|
||||
from app.domain.taxonomy import ErrorCode, FindingSource
|
||||
from app.scenarios.loader import load_file
|
||||
from app.scoring.grammar import GrammarAssessment
|
||||
from app.session import finish as finish_module
|
||||
from app.session.dds import prepare_handoff_queue, prepare_queue
|
||||
from app.session.hub import hub
|
||||
from app.session.score import ScoringSnapshot, score_session
|
||||
from app.session.state import SessionState
|
||||
from app.session.store import MemorySessionStore
|
||||
|
||||
LIBRARY = Path(__file__).resolve().parents[2] / "scenarios"
|
||||
|
||||
|
||||
def scenario(name: str):
|
||||
return load_file(LIBRARY / name, LIBRARY)
|
||||
|
||||
|
||||
def card_state(*, grammar: bool = True, handoff: bool = False) -> SessionState:
|
||||
operator = scenario("fire-apartment-l2.yaml")
|
||||
state = SessionState(
|
||||
session_id=uuid4(), scenario_id=operator.id, scenario_title=operator.title,
|
||||
level=operator.level.value, mode=SessionMode.TRAINING, exercise=Exercise.CARD,
|
||||
handoff_to_dds=handoff, scenario=operator,
|
||||
criteria=LessonCriteria(require_correct_grammar=grammar),
|
||||
)
|
||||
state.kio = KIO(
|
||||
address="улица Ленина, 14", incident_type="fire", victims_count=2,
|
||||
description="горит балкон", notify=["Служба 101"],
|
||||
)
|
||||
state.on_event("card.start")
|
||||
state.dispatch()
|
||||
state.on_event("card.submit")
|
||||
if handoff:
|
||||
prepare_handoff_queue(state, [scenario("tickets/t01-1-fire-container.yaml")])
|
||||
return state
|
||||
|
||||
|
||||
def dds_state() -> SessionState:
|
||||
first = scenario("fire-apartment-l2.yaml")
|
||||
state = SessionState(
|
||||
session_id=uuid4(), scenario_id=first.id, scenario_title=first.title,
|
||||
level=first.level.value, mode=SessionMode.TRAINING, exercise=Exercise.DDS,
|
||||
criteria=LessonCriteria(require_correct_grammar=True),
|
||||
)
|
||||
prepare_queue(state, [first, scenario("tickets/t20-2-stroke.yaml")])
|
||||
return state
|
||||
|
||||
|
||||
def grammar_says(passed: bool, calls: list[str]):
|
||||
async def check(text: str) -> GrammarAssessment:
|
||||
calls.append(text)
|
||||
return GrammarAssessment(
|
||||
passed=passed, errors=() if passed else ("нет завершающего знака препинания",),
|
||||
source="test",
|
||||
)
|
||||
return check
|
||||
|
||||
|
||||
def score(snapshot, grammar):
|
||||
return asyncio.run(score_session(snapshot, grammar=grammar))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def offline_finish(monkeypatch):
|
||||
class NoCoaching:
|
||||
def model_dump(self, **_kwargs):
|
||||
return {}
|
||||
|
||||
async def no_coach(_metrics):
|
||||
return NoCoaching()
|
||||
|
||||
monkeypatch.setattr(finish_module, "coach", no_coach)
|
||||
monkeypatch.setattr(hub, "store", MemorySessionStore())
|
||||
|
||||
|
||||
def test_passed_grammar_check_adds_passed_metric_without_e4():
|
||||
calls: list[str] = []
|
||||
snapshot = ScoringSnapshot.of(card_state())
|
||||
|
||||
report = score(snapshot, grammar_says(True, calls))
|
||||
|
||||
assert calls == ["горит балкон"]
|
||||
metric = next(item for item in report.result.metrics if item.key == "description_grammar")
|
||||
assert metric.passed and metric.fact == "ошибок не обнаружено"
|
||||
assert ErrorCode.E4 not in [item.code for item in report.result.findings]
|
||||
|
||||
|
||||
def test_failed_grammar_check_adds_e4_and_lowers_score():
|
||||
passed = score(ScoringSnapshot.of(card_state()), grammar_says(True, []))
|
||||
failed = score(ScoringSnapshot.of(card_state()), grammar_says(False, []))
|
||||
|
||||
metric = next(item for item in failed.result.metrics if item.key == "description_grammar")
|
||||
assert not metric.passed
|
||||
assert metric.fact == "нет завершающего знака препинания"
|
||||
e4 = next(item for item in failed.result.findings if item.code is ErrorCode.E4)
|
||||
assert e4.source is FindingSource.GRAMMAR
|
||||
assert failed.result.score < passed.result.score
|
||||
|
||||
|
||||
def test_grammar_is_not_checked_without_criterion_or_on_dds_exercise():
|
||||
calls: list[str] = []
|
||||
|
||||
off = score(ScoringSnapshot.of(card_state(grammar=False)), grammar_says(False, calls))
|
||||
dds = score(ScoringSnapshot.of(dds_state()), grammar_says(False, calls))
|
||||
|
||||
assert calls == []
|
||||
assert "description_grammar" not in [item.key for item in off.result.metrics]
|
||||
assert "description_grammar" not in [item.key for item in dds.result.metrics]
|
||||
|
||||
|
||||
def test_snapshot_is_read_only():
|
||||
snapshot = ScoringSnapshot.of(card_state())
|
||||
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
snapshot.scenario = None
|
||||
|
||||
|
||||
def test_card_fill_time_is_scored_from_snapshot_timers():
|
||||
report = score(ScoringSnapshot.of(card_state(grammar=False)), grammar_says(True, []))
|
||||
|
||||
metric = next(item for item in report.result.metrics if item.key == "card_fill_time")
|
||||
assert metric.passed
|
||||
assert metric.fact == "0 с (на 180 с быстрее норматива)"
|
||||
assert report.summary["card_fill_ms"] is not None
|
||||
|
||||
|
||||
def test_dds_snapshot_scores_every_queue_card_and_leaves_state_untouched():
|
||||
state = dds_state()
|
||||
finish_module.score_live_dds_cards(state)
|
||||
completed = list(state.desk.completed)
|
||||
snapshot = ScoringSnapshot.of(state)
|
||||
|
||||
report = score(snapshot, grammar_says(True, []))
|
||||
|
||||
assert [card.scenario_id for card in report.cards] == ["fire-apartment-l2", "t20-2-stroke"]
|
||||
assert [item.title for item in report.result.metrics] == [
|
||||
item.title for card in completed for item in card.metrics
|
||||
]
|
||||
assert state.desk.completed == completed
|
||||
|
||||
|
||||
def test_handoff_live_and_refreshed_reports_use_operator_scenario_and_same_score():
|
||||
state = card_state(handoff=True)
|
||||
active = state.desk.ordered()[1]
|
||||
state.desk.open(active.card_id)
|
||||
active.set_service_status(
|
||||
active.dispatched_card.notify[0], ServiceStatus.ACCEPTED,
|
||||
"Основание: доклад.\nСведения: принято.",
|
||||
)
|
||||
# Предусловие ошибки: к завершению активна не карточка курсанта.
|
||||
assert state.desk.active.scenario.id == "t01-1-fire-container"
|
||||
assert ScoringSnapshot.of(state).scenario.id == "fire-apartment-l2"
|
||||
|
||||
async def live_then_refreshed():
|
||||
async with hub.operation(state.session_id):
|
||||
await finish_module.finish(state.session_id, state)
|
||||
live = dict(state.score["full_report"])
|
||||
state.self_assessed = True
|
||||
async with hub.operation(state.session_id):
|
||||
await finish_module.refresh_archived_report(state.session_id, state)
|
||||
return live, state.score["full_report"]
|
||||
|
||||
live, refreshed = asyncio.run(live_then_refreshed())
|
||||
|
||||
assert live["scenario_id"] == refreshed["scenario_id"] == "fire-apartment-l2"
|
||||
assert live["score_auto"] == refreshed["score_auto"] == state.score["score_auto"]
|
||||
assert live["metrics"] == refreshed["metrics"]
|
||||
Loading…
Reference in a new issue