diff --git a/backend/app/api/http/sessions.py b/backend/app/api/http/sessions.py index 9f3defd..c22ddb6 100644 --- a/backend/app/api/http/sessions.py +++ b/backend/app/api/http/sessions.py @@ -29,7 +29,8 @@ from app.scoring.export import to_csv, to_pdf from app.scoring.report import build as build_report from app.session.access import can_access from app.session.checkpoint import load_state -from app.session.finish import override_score, scoring_scenario +from app.session.finish import override_score +from app.session.score import scoring_scenario from app.session.hub import hub from app.session.store import ScoreOverridden, apply_score_override from app.voice.recording import recording_path diff --git a/backend/app/session/finish.py b/backend/app/session/finish.py index 405183d..7d7f64a 100644 --- a/backend/app/session/finish.py +++ b/backend/app/session/finish.py @@ -12,22 +12,18 @@ import logging import time from uuid import UUID -from app.domain.events import CallEnded, CallEndReason, Exercise, Metric, ScoreReady, SessionEnded +from app.domain.events import CallEnded, CallEndReason, Exercise, ScoreReady, SessionEnded from app.domain.statuses import ServiceStatus, current -from app.domain.taxonomy import Competency, ErrorCode, Finding, FindingSource from app.domain.timers import TimerCode -from app.scenarios import store from app.scoring.ai_coach import coach -from app.scoring.card import evaluate_card -from app.scoring.competency import radar from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher -from app.scoring.gost import GostResult, evaluate +from app.scoring.gost import GostResult from app.scoring.grammar import assess from app.scoring.report import build as build_report -from app.scoring.taxonomy import METRIC_WEIGHTS -from app.scoring.timing import time_credit +from app.scoring.timing import DDS_WORK_TIME, time_metric from app.scoring.weights import apply_weights from app.session.hub import hub +from app.session.score import ScoringSnapshot, score_session, scoring_scenario from app.session.state import DdsCardRecord, DdsLiveCard, now_utc from app.session.store import ( LessonEnded, @@ -77,39 +73,12 @@ def score_dds_card(state, card: DdsLiveCard) -> DdsCardRecord: } for service in services ) - work_passed = terminal and work_elapsed_ms is not None and work_elapsed_ms <= work_limit_ms - work_delta_ms = (work_elapsed_ms - work_limit_ms) if work_elapsed_ms is not None else None - if work_elapsed_ms is None: - work_fact = "время обработки не зафиксировано" - elif work_delta_ms and work_delta_ms > 0: - work_fact = f"{round(work_elapsed_ms / 1000)} с (+{round(work_delta_ms / 1000)} с сверх норматива)" - elif work_delta_ms and work_delta_ms < 0: - work_fact = (f"{round(work_elapsed_ms / 1000)} с (на " - f"{round(abs(work_delta_ms) / 1000)} с быстрее норматива)") - else: - work_fact = f"{round(work_elapsed_ms / 1000)} с (точно в норматив)" - if not terminal: - work_fact = f"карточка не завершена; {work_fact}" - weighted.metrics.append(Metric( - key="dds_work_time", - title="Отработка карточки ДДС", - fact=work_fact, - norm=f"завершить за {round(work_limit_ms / 1000)} с", - ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026", - passed=work_passed, - weight=METRIC_WEIGHTS["dds_work_time"], - credit=time_credit(work_elapsed_ms, work_limit_ms) if terminal else 0.0, - )) - if not work_passed: - weighted.findings.append(Finding( - code=ErrorCode.E3, - source=FindingSource.TIMERS, - summary="ДДС: норматив времени отработки карточки не выполнен", - fact=work_fact, - norm=f"завершить за {round(work_limit_ms / 1000)} с", - ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026", - competency=Competency.NORMS, - )) + work_metric, work_finding = time_metric( + DDS_WORK_TIME, work_elapsed_ms, work_limit_ms, completed=terminal + ) + weighted.metrics.append(work_metric) + if work_finding is not None: + weighted.findings.append(work_finding) apply_weights(weighted, {**card.scenario.score_weights, **state.criteria.score_weights}) actions = [ {"type": "card.status", "service": mark.service, "status": mark.status.value, @@ -149,20 +118,6 @@ def score_dds_card(state, card: DdsLiveCard) -> DdsCardRecord: ) -def scoring_scenario(state): - """Сценарий, по которому оценивается и разбирается занятие. - - После передачи 112→ДДС КИО курсанта оценивается по эталону упражнения - 112. Балл, разбор и чек-лист самооценки обязаны брать один и тот же - сценарий. - """ - if state.handoff_to_dds and state.operator_scenario: - return state.operator_scenario - if state.scenario is None and state.desk.scenarios: - return state.desk.scenarios[0] - return state.scenario or store.get(state.scenario_id) - - def score_live_dds_cards(state) -> list[DdsCardRecord]: """Все карточки занятия: завершённые и оценка ещё открытых. @@ -182,185 +137,30 @@ async def finish(session_id: UUID, state) -> None: if state.recorder is not None: path = await asyncio.to_thread(state.recorder.finalize) state.recording_path = str(path) if path else None + # Все карточки выданы одновременно: при досрочном завершении оцениваем + # каждую, включая не открытую, потому что её норматив уже шёл. + if state.exercise is Exercise.DDS or (state.handoff_to_dds and state.desk.scenarios): + score_live_dds_cards(state) # Сценарий занятия, а не библиотечный: директивы могли поправить эталон. - scenario = scoring_scenario(state) - if scenario is None: + snapshot = ScoringSnapshot.of(state) + if snapshot is None: return - - cards: list[DdsCardRecord] = [] - if state.exercise is Exercise.CARD: - result = evaluate_card( - scenario, state.operator_kio or state.dispatched_card or state.kio - ) - limit_ms = state.timers.limits[TimerCode.CARD_FILL] - elapsed_ms = state.timers.measured_ms(TimerCode.CARD_FILL) - submitted = state.dispatched_card is not None - if elapsed_ms is None: - elapsed_fact = "время не зафиксировано" - else: - delta_ms = elapsed_ms - limit_ms - elapsed_seconds = round(elapsed_ms / 1000) - if delta_ms > 0: - deviation = f"+{round(delta_ms / 1000)} с сверх норматива" - elif delta_ms < 0: - deviation = f"на {round(abs(delta_ms) / 1000)} с быстрее норматива" - else: - deviation = "точно в норматив" - elapsed_fact = f"{elapsed_seconds} с ({deviation})" - passed = submitted and elapsed_ms is not None and elapsed_ms <= limit_ms - result.metrics.append(Metric( - key="card_fill_time", - title="Время заполнения карточки", - fact=elapsed_fact if submitted else f"карточка не сдана; {elapsed_fact}", - norm=f"сдать карточку за {round(limit_ms / 1000)} с", - ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026", - passed=passed, - weight=METRIC_WEIGHTS["card_fill_time"], - credit=time_credit(elapsed_ms, limit_ms) if submitted else 0.0, - )) - if not passed: - result.findings.append(Finding( - code=ErrorCode.E3, - source=FindingSource.TIMERS, - summary="Время заполнения карточки: норматив не выполнен", - fact=elapsed_fact if submitted else f"карточка не сдана; {elapsed_fact}", - norm=f"сдать карточку за {round(limit_ms / 1000)} с", - ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026", - competency=Competency.NORMS, - )) - elif state.exercise is Exercise.DDS: - # Все карточки выданы одновременно: при досрочном завершении оцениваем - # каждую, включая не открытую, потому что её норматив уже шёл. - result = GostResult() - cards = score_live_dds_cards(state) - for card in cards: - result.metrics.extend(card.metrics) - result.findings.extend(card.findings) - else: - result = evaluate( - scenario=scenario, - kio=state.kio, - timers=state.timers, - revealed_facts=[fact.id for fact in state.slots.revealed_facts()] if state.slots else None, - refined_facts=list(state.slots.refined) if state.slots else None, - end_reason=state.end_reason, - bounced_fields=state.bounced_fields, - resolved_outcome=state.resolved_outcome, - dispatched=state.dispatched_card is not None, - ) - # Грамматика относится к свободному описанию оператора 112, а не к - # заполнению карточки на стороне ДДС. Учитывается только по явной настройке - # преподавателя и только когда есть текст для проверки. - operator_kio = state.operator_kio if state.handoff_to_dds else None - description = (operator_kio or state.kio).description or "" - if (state.exercise is not Exercise.DDS and state.criteria.require_correct_grammar - and description.strip()): - grammar = await assess(description) - result.metrics.append(Metric( - key="description_grammar", - title="Грамматика описания происшествия", - fact="ошибок не обнаружено" if grammar.passed else "; ".join(grammar.errors), - norm="грамматически корректное описание", - ref="критерий занятия; правила русского языка", - passed=grammar.passed, - weight=1.0, - )) - if not grammar.passed: - result.findings.append(Finding( - code=ErrorCode.E4, - source=FindingSource.GRAMMAR, - summary="Грамматическая ошибка в описании происшествия", - fact="; ".join(grammar.errors), - norm="грамматически корректное описание", - ref="критерий занятия; правила русского языка", - competency=Competency.COMMUNICATION, - )) - # Работа диспетчера — вторая роль и вторая таксономия. Отметки D1–D6 идут - # рядом с E1–E6, а не вместо: в живой цепочке 112 → ДДС в одном занятии - # участвуют оба (docs/spec/DATASET.md#статусы-реагирования). - live_card = state.desk.active - if live_card is not None and state.exercise is Exercise.CALL: - decision_limit_ms = live_card.timers.limits[TimerCode.DDS_ACK] - services = state.card_services(live_card) - dispatcher_findings = evaluate_dispatcher( - entries=live_card.status_log, - services=services, - crew_assignments=live_card.crew_assignments, - deadline_ms=decision_limit_ms, - elapsed_ms=live_card.timers.measured_ms(TimerCode.DDS_ACK), - reply_text=live_card.reply_text, - expected_decision=scenario.dds_decision.expected, - expected_decision_reason=scenario.dds_decision.reason, - ) - result.findings.extend(dispatcher_findings) - result.metrics.extend(dispatcher_metrics( - live_card, decision_limit_ms, scenario.dds_decision.expected, - scenario.dds_decision.reason, services=services, - )) - # DDS cards were weighted individually in score_dds_card using each - # card's scenario defaults plus the lesson override. Reapplying the first - # scenario's weights here would corrupt the other ticket cards. - if state.exercise is not Exercise.DDS: - apply_weights(result, {**scenario.score_weights, **state.criteria.score_weights}) - - if state.handoff_to_dds and state.desk.scenarios: - # В связке КИО оценивается относительно эталона и весов упражнения - # 112, а каждая карточка очереди уже взвешена собственным сценарием. - cards = score_live_dds_cards(state) - for card in cards: - result.metrics.extend(card.metrics) - result.findings.extend(card.findings) - - # Сводка числами: по ней считается дельта между попытками в профиле. - # Вытаскивать её разбором текста метрик («94 с») — путь к тихим ошибкам. - required = scenario.ground_truth.required_facts - revealed = [fact.id for fact in state.slots.revealed_facts()] if state.slots else [] - codes: dict[str, int] = {} - for finding in result.findings: - codes[finding.code.value] = codes.get(finding.code.value, 0) + 1 - - state.score = { - "score_auto": result.score, - "summary": { - "interview_ms": state.timers.measured_ms(TimerCode.INTERVIEW), - "card_fill_ms": state.timers.measured_ms(TimerCode.CARD_FILL), - "facts_got": len([fact for fact in required if fact in revealed]), - "facts_required": len(required), - "hints": len(state.hints_shown), - "codes": codes, - }, - "metrics": [metric.model_dump() for metric in result.metrics], - "findings": [finding.model_dump(mode="json") for finding in result.findings], - "competencies": [item.model_dump() for item in radar(result.metrics)], - "unavailable": result.unavailable, - # Статус карточки — готовая красная метка, понятная любому диспетчеру. - "card_status": state.station_snapshot().card.value, - "card_results": [ - {"card_id": str(card.card_id), "scenario_id": card.scenario_id, - "score_auto": card.score_auto, "reply_text": card.reply_text, - "actions": card.actions, "duration_ms": card.duration_ms, - "title": card.title, "address": card.address, - "description": card.description, "incident_type": card.incident_type, - "victims_count": card.victims_count, - "received_at": card.received_at.isoformat() if card.received_at else None, - "managed_service": card.managed_service, - "recipient_services": card.recipient_services, - "metrics": [metric.model_dump(mode="json") for metric in card.metrics], - "findings": [finding.model_dump(mode="json") for finding in card.findings]} - for card in cards - ] if state.exercise is Exercise.DDS or state.handoff_to_dds else [], - } + report = await score_session(snapshot, grammar=assess) + state.score = report.payload(card_status=state.station_snapshot().card.value) # Модель только поясняет уже посчитанные провалы и балл не трогает; без # запущенной модели разбор выходит со статусом «недоступно», а не ждёт её. - state.score["ai_coaching"] = (await coach(result.metrics)).model_dump(mode="json") + state.score["ai_coaching"] = (await coach(report.result.metrics)).model_dump(mode="json") # Полный разбор хранится вместе с оценкой: PDF/CSV и история должны # переживать перезапуск backend, а не зависеть от живого объекта в реестре хаба. - state.score["full_report"] = build_report(session_id, state, scenario).model_dump(mode="json") - log.info("сессия %s: оценка %.1f, отметок %d", session_id, result.score, len(result.findings)) + state.score["full_report"] = build_report( + session_id, state, snapshot.scenario + ).model_dump(mode="json") + log.info("сессия %s: оценка %.1f, отметок %d", + session_id, report.result.score, len(report.result.findings)) # Оценка и аудит уходят в commit операции, завершившей занятие; ScoreReady # рассылает `end_session` и ждёт того же commit — без записи итог не выдаётся. - hub.record(session_id, ScoreCalculated(result.score, state.score)) + hub.record(session_id, ScoreCalculated(report.result.score, state.score)) def _ends_on_station(state) -> bool: diff --git a/backend/app/session/score.py b/backend/app/session/score.py new file mode 100644 index 0000000..10c637c --- /dev/null +++ b/backend/app/session/score.py @@ -0,0 +1,263 @@ +"""Оценка занятия по неизменяемому снимку. + +`finish()` сначала доводит живое занятие до оцениваемого вида (дооценивает +очередь ДДС), затем снимает `ScoringSnapshot` и отдаёт его `score_session`. +Сам расчёт не читает и не меняет `state`: живой, сохранённый и перестроенный +отчёт получают сценарий одним путём — через `scoring_scenario`. +""" + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any + +from app.domain.events import CallEndReason, Exercise, LessonCriteria, Metric +from app.domain.kio import KIO +from app.domain.taxonomy import Competency, ErrorCode, Finding, FindingSource +from app.domain.timers import TimerCode +from app.scenarios import store +from app.scenarios.schema import Scenario +from app.scoring.card import evaluate_card +from app.scoring.competency import radar +from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher +from app.scoring.gost import GostResult, evaluate +from app.scoring.grammar import GrammarAssessment, assess +from app.scoring.timing import CARD_FILL_TIME, time_metric +from app.scoring.weights import apply_weights +from app.session.dds import DdsCardRecord, DdsLiveCard +from app.session.timers import SessionTimers + +GrammarCheck = Callable[[str], Awaitable[GrammarAssessment]] + + +def scoring_scenario(state) -> Scenario | None: + """Сценарий, по которому оценивается и разбирается занятие. + + После передачи 112→ДДС КИО курсанта оценивается по эталону упражнения + 112. Балл, разбор и чек-лист самооценки обязаны брать один и тот же + сценарий. + """ + if state.handoff_to_dds and state.operator_scenario: + return state.operator_scenario + if state.scenario is None and state.desk.scenarios: + return state.desk.scenarios[0] + return state.scenario or store.get(state.scenario_id) + + +@dataclass(frozen=True) +class ScoringSnapshot: + """Всё, что нужно для балла, снятое с занятия в момент завершения.""" + + scenario: Scenario + exercise: Exercise + handoff_to_dds: bool + #: КИО, которую сверяют с эталоном: в упражнении 112 — сданная карточка. + kio: KIO + #: Свободное описание оператора 112 — предмет проверки грамматики. + description: str + dispatched: bool + timers: SessionTimers + criteria: LessonCriteria + revealed_facts: tuple[str, ...] | None + refined_facts: tuple[str, ...] | None + end_reason: CallEndReason | None + bounced_fields: tuple[str, ...] + resolved_outcome: str | None + hints: int + #: Уже оценённые карточки очереди ДДС, в порядке выдачи. + dds_cards: tuple[DdsCardRecord, ...] + #: Карточка на пульте в упражнении по звонку: отметки D1–D6 рядом с E1–E6. + call_card: DdsLiveCard | None + call_card_services: tuple[str, ...] + + @property + def reports_cards(self) -> bool: + return self.exercise is Exercise.DDS or self.handoff_to_dds + + @classmethod + def of(cls, state) -> "ScoringSnapshot | None": + """Снимок живого занятия; None — оценивать не по чему. + + Очередь ДДС к этому моменту должна быть дооценена + (`finish.score_live_dds_cards`): снимок берёт готовые записи. + """ + scenario = scoring_scenario(state) + if scenario is None: + return None + if state.exercise is Exercise.CARD: + kio = state.operator_kio or state.dispatched_card or state.kio + else: + kio = state.kio + # Грамматика относится к свободному описанию оператора 112, а не к + # заполнению карточки на стороне ДДС. + operator_kio = state.operator_kio if state.handoff_to_dds else None + live_card = state.desk.active if state.exercise is Exercise.CALL else None + queue = state.exercise is Exercise.DDS or (state.handoff_to_dds and state.desk.scenarios) + return cls( + scenario=scenario, + exercise=state.exercise, + handoff_to_dds=state.handoff_to_dds, + kio=kio.model_copy(deep=True), + description=(operator_kio or state.kio).description or "", + dispatched=state.dispatched_card is not None, + timers=state.timers.model_copy(deep=True), + criteria=state.criteria.model_copy(deep=True), + revealed_facts=(tuple(fact.id for fact in state.slots.revealed_facts()) + if state.slots else None), + refined_facts=tuple(state.slots.refined) if state.slots else None, + end_reason=state.end_reason, + bounced_fields=tuple(state.bounced_fields), + resolved_outcome=state.resolved_outcome, + hints=len(state.hints_shown), + dds_cards=tuple(state.desk.completed) if queue else (), + call_card=live_card.model_copy(deep=True) if live_card is not None else None, + call_card_services=(tuple(state.card_services(live_card)) + if live_card is not None else ()), + ) + + +@dataclass(frozen=True) +class ScoreReport: + """Посчитанная оценка: результат ГОСТ, сводка числами и карточки ДДС.""" + + result: GostResult + summary: dict[str, Any] + cards: tuple[DdsCardRecord, ...] + reports_cards: bool + + def payload(self, *, card_status: str) -> dict[str, Any]: + """Сохраняемая оценка занятия — без разбора модели и полного отчёта.""" + return { + "score_auto": self.result.score, + "summary": self.summary, + "metrics": [metric.model_dump() for metric in self.result.metrics], + "findings": [finding.model_dump(mode="json") for finding in self.result.findings], + "competencies": [item.model_dump() for item in radar(self.result.metrics)], + "unavailable": self.result.unavailable, + # Статус карточки — готовая красная метка, понятная любому диспетчеру. + "card_status": card_status, + "card_results": [ + {"card_id": str(card.card_id), "scenario_id": card.scenario_id, + "score_auto": card.score_auto, "reply_text": card.reply_text, + "actions": card.actions, "duration_ms": card.duration_ms, + "title": card.title, "address": card.address, + "description": card.description, "incident_type": card.incident_type, + "victims_count": card.victims_count, + "received_at": card.received_at.isoformat() if card.received_at else None, + "managed_service": card.managed_service, + "recipient_services": card.recipient_services, + "metrics": [metric.model_dump(mode="json") for metric in card.metrics], + "findings": [finding.model_dump(mode="json") for finding in card.findings]} + for card in self.cards + ] if self.reports_cards else [], + } + + +async def score_session( + snapshot: ScoringSnapshot, *, grammar: GrammarCheck = assess +) -> ScoreReport: + """Балл занятия. Единственный внешний вызов — проверка грамматики, и та + только по явному критерию преподавателя.""" + scenario = snapshot.scenario + if snapshot.exercise is Exercise.CARD: + result = evaluate_card(scenario, snapshot.kio) + metric, finding = time_metric( + CARD_FILL_TIME, + snapshot.timers.measured_ms(TimerCode.CARD_FILL), + snapshot.timers.limits[TimerCode.CARD_FILL], + completed=snapshot.dispatched, + ) + result.metrics.append(metric) + if finding is not None: + result.findings.append(finding) + elif snapshot.exercise is Exercise.DDS: + # Все карточки выданы одновременно: при досрочном завершении оценена + # каждая, включая не открытую, потому что её норматив уже шёл. + result = GostResult() + else: + result = evaluate( + scenario=scenario, + kio=snapshot.kio, + timers=snapshot.timers, + revealed_facts=(list(snapshot.revealed_facts) + if snapshot.revealed_facts is not None else None), + refined_facts=(list(snapshot.refined_facts) + if snapshot.refined_facts is not None else None), + end_reason=snapshot.end_reason, + bounced_fields=list(snapshot.bounced_fields), + resolved_outcome=snapshot.resolved_outcome, + dispatched=snapshot.dispatched, + ) + # Учитывается только по явной настройке преподавателя и только когда есть + # текст для проверки. + if (snapshot.exercise is not Exercise.DDS and snapshot.criteria.require_correct_grammar + and snapshot.description.strip()): + verdict = await grammar(snapshot.description) + result.metrics.append(Metric( + key="description_grammar", + title="Грамматика описания происшествия", + fact="ошибок не обнаружено" if verdict.passed else "; ".join(verdict.errors), + norm="грамматически корректное описание", + ref="критерий занятия; правила русского языка", + passed=verdict.passed, + weight=1.0, + )) + if not verdict.passed: + result.findings.append(Finding( + code=ErrorCode.E4, + source=FindingSource.GRAMMAR, + summary="Грамматическая ошибка в описании происшествия", + fact="; ".join(verdict.errors), + norm="грамматически корректное описание", + ref="критерий занятия; правила русского языка", + competency=Competency.COMMUNICATION, + )) + # Работа диспетчера — вторая роль и вторая таксономия. Отметки D1–D6 идут + # рядом с E1–E6, а не вместо: в живой цепочке 112 → ДДС в одном занятии + # участвуют оба (docs/spec/DATASET.md#статусы-реагирования). + live_card = snapshot.call_card + if live_card is not None: + decision_limit_ms = live_card.timers.limits[TimerCode.DDS_ACK] + services = list(snapshot.call_card_services) + result.findings.extend(evaluate_dispatcher( + entries=live_card.status_log, + services=services, + crew_assignments=live_card.crew_assignments, + deadline_ms=decision_limit_ms, + elapsed_ms=live_card.timers.measured_ms(TimerCode.DDS_ACK), + reply_text=live_card.reply_text, + expected_decision=scenario.dds_decision.expected, + expected_decision_reason=scenario.dds_decision.reason, + )) + result.metrics.extend(dispatcher_metrics( + live_card, decision_limit_ms, scenario.dds_decision.expected, + scenario.dds_decision.reason, services=services, + )) + # Карточки ДДС уже взвешены каждая своим сценарием плюс настройкой + # занятия. Веса первого сценария поверх них испортили бы остальные. + if snapshot.exercise is not Exercise.DDS: + apply_weights(result, {**scenario.score_weights, **snapshot.criteria.score_weights}) + # В связке КИО оценивается относительно эталона и весов упражнения 112, + # а каждая карточка очереди уже взвешена собственным сценарием. + for card in snapshot.dds_cards: + result.metrics.extend(card.metrics) + result.findings.extend(card.findings) + + # Сводка числами: по ней считается дельта между попытками в профиле. + # Вытаскивать её разбором текста метрик («94 с») — путь к тихим ошибкам. + required = scenario.ground_truth.required_facts + revealed = snapshot.revealed_facts or () + codes: dict[str, int] = {} + for finding in result.findings: + codes[finding.code.value] = codes.get(finding.code.value, 0) + 1 + summary = { + "interview_ms": snapshot.timers.measured_ms(TimerCode.INTERVIEW), + "card_fill_ms": snapshot.timers.measured_ms(TimerCode.CARD_FILL), + "facts_got": len([fact for fact in required if fact in revealed]), + "facts_required": len(required), + "hints": snapshot.hints, + "codes": codes, + } + return ScoreReport( + result=result, summary=summary, cards=snapshot.dds_cards, + reports_cards=snapshot.reports_cards, + ) diff --git a/backend/tests/test_dispatcher_scoring.py b/backend/tests/test_dispatcher_scoring.py index ee8c0e6..8bc5c43 100644 --- a/backend/tests/test_dispatcher_scoring.py +++ b/backend/tests/test_dispatcher_scoring.py @@ -1,12 +1,28 @@ from datetime import datetime, timezone -from types import SimpleNamespace +from pathlib import Path import pytest from pydantic import ValidationError from app.domain.statuses import ServiceStatus, StatusEntry from app.domain.taxonomy import ErrorCode +from app.scenarios.loader import load_file from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher +from app.session.dds import DdsLiveCard, build_card +from app.session.timers import SessionTimers + +LIBRARY = Path(__file__).resolve().parents[2] / "scenarios" +SERVICES = ["Служба 101"] + + +def live_card(*, status_log, crew_assignments, dispatched_at) -> DdsLiveCard: + """Карточка пульта с заданным журналом статусов — вход оценки ДДС.""" + scenario = load_file(LIBRARY / "fire-apartment-l2.yaml", LIBRARY) + card = build_card(scenario, 0, SessionTimers().limits) + card.status_log = status_log + card.crew_assignments = crew_assignments + card.dispatched_at = dispatched_at + return card def test_late_primary_status_has_d1_finding_even_when_status_exists(): @@ -110,8 +126,7 @@ def test_missing_manual_status_comment_has_a_d5_finding(): def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric(): at = datetime.now(timezone.utc) - state = SimpleNamespace( - managed_services=lambda: ["Служба 101"], + card = live_card( status_log=[ StatusEntry( service="Служба 101", status=ServiceStatus.ACCEPTED, @@ -127,7 +142,7 @@ def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric(): ) reply = next( - metric for metric in dispatcher_metrics(state, 30_000) + metric for metric in dispatcher_metrics(card, 30_000, services=SERVICES) if metric.key == "dds_reply" ) @@ -135,9 +150,9 @@ def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric(): assert "к каждой ручной отметке" in reply.norm findings = evaluate_dispatcher( - entries=state.status_log, + entries=card.status_log, services=["Служба 101"], - crew_assignments=state.crew_assignments, + crew_assignments=card.crew_assignments, deadline_ms=30_000, elapsed_ms=0, ) @@ -173,13 +188,13 @@ def test_scenario_may_define_a_valid_decline_for_duplicate_or_territory(): expected_decision_reason="дублирующая карточка", ) assert ErrorCode.D3 not in [finding.code for finding in findings] - state = SimpleNamespace( - managed_services=lambda: ["Служба 101"], status_log=entries, + card = live_card( + status_log=entries, crew_assignments={}, dispatched_at=at, ) metrics = dispatcher_metrics( - state, 30_000, expected_decision="decline", - expected_decision_reason="дублирующая карточка", + card, 30_000, expected_decision="decline", + expected_decision_reason="дублирующая карточка", services=SERVICES, ) decision = next(metric for metric in metrics if metric.key == "dds_decision") assert decision.passed @@ -188,8 +203,7 @@ def test_scenario_may_define_a_valid_decline_for_duplicate_or_territory(): def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes(): at = datetime.now(timezone.utc) - state = SimpleNamespace( - managed_services=lambda: ["Служба 101"], + card = live_card( status_log=[StatusEntry( service="Служба 101", status=ServiceStatus.ACCEPTED, at=at, comment="бригада на связи", @@ -197,19 +211,19 @@ def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes( crew_assignments={"Служба 101": "Бригада 12"}, dispatched_at=at, ) - reply = next(m for m in dispatcher_metrics(state, 30_000) if m.key == "dds_reply") + reply = next(m for m in dispatcher_metrics(card, 30_000, services=SERVICES) if m.key == "dds_reply") assert not reply.passed findings = evaluate_dispatcher( - entries=state.status_log, services=["Служба 101"], - crew_assignments=state.crew_assignments, + entries=card.status_log, services=["Служба 101"], + crew_assignments=card.crew_assignments, deadline_ms=30_000, elapsed_ms=0, ) assert any("не разделяют основание и сведения" in f.summary for f in findings) - state.status_log[0] = state.status_log[0].model_copy(update={ + card.status_log[0] = card.status_log[0].model_copy(update={ "comment": "Основание: доклад старшего.\nСведения: бригада на связи.", }) - reply = next(m for m in dispatcher_metrics(state, 30_000) if m.key == "dds_reply") + reply = next(m for m in dispatcher_metrics(card, 30_000, services=SERVICES) if m.key == "dds_reply") assert reply.passed def test_accepting_card_with_scenario_expected_decline_is_explained(): @@ -242,11 +256,11 @@ def test_incomplete_work_path_has_d6_for_failed_progress_metrics(): StatusEntry(service="Служба 101", status=ServiceStatus.ACCEPTED, at=at), StatusEntry(service="Служба 101", status=ServiceStatus.RESPONDING, at=at), ] - state = SimpleNamespace( - managed_services=lambda: ["Служба 101"], status_log=entries, + card = live_card( + status_log=entries, crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at, ) - metrics = dispatcher_metrics(state, 30_000) + metrics = dispatcher_metrics(card, 30_000, services=SERVICES) assert not next(item for item in metrics if item.key == "dds_progress").passed assert not next(item for item in metrics if item.key == "dds_completion").passed findings = evaluate_dispatcher( @@ -267,11 +281,11 @@ def test_reasoned_refusal_after_acceptance_is_a_valid_terminal_path(): comment="Бригаде переданы сведения, выезд не выполнялся по причине угрозы.", ), ] - state = SimpleNamespace( - managed_services=lambda: ["Служба 101"], status_log=entries, + card = live_card( + status_log=entries, crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at, ) - metrics = dispatcher_metrics(state, 30_000) + metrics = dispatcher_metrics(card, 30_000, services=SERVICES) assert next(item for item in metrics if item.key == "dds_progress").passed assert next(item for item in metrics if item.key == "dds_completion").passed findings = evaluate_dispatcher( diff --git a/backend/tests/test_score_session.py b/backend/tests/test_score_session.py new file mode 100644 index 0000000..cb20fa6 --- /dev/null +++ b/backend/tests/test_score_session.py @@ -0,0 +1,179 @@ +"""Оценка занятия по снимку: без websocket, без живого `state` внутри расчёта.""" + +import asyncio +from dataclasses import FrozenInstanceError +from pathlib import Path +from uuid import uuid4 + +import pytest + +from app.domain.events import Exercise, LessonCriteria, SessionMode +from app.domain.kio import KIO +from app.domain.statuses import ServiceStatus +from app.domain.taxonomy import ErrorCode, FindingSource +from app.scenarios.loader import load_file +from app.scoring.grammar import GrammarAssessment +from app.session import finish as finish_module +from app.session.dds import prepare_handoff_queue, prepare_queue +from app.session.hub import hub +from app.session.score import ScoringSnapshot, score_session +from app.session.state import SessionState +from app.session.store import MemorySessionStore + +LIBRARY = Path(__file__).resolve().parents[2] / "scenarios" + + +def scenario(name: str): + return load_file(LIBRARY / name, LIBRARY) + + +def card_state(*, grammar: bool = True, handoff: bool = False) -> SessionState: + operator = scenario("fire-apartment-l2.yaml") + state = SessionState( + session_id=uuid4(), scenario_id=operator.id, scenario_title=operator.title, + level=operator.level.value, mode=SessionMode.TRAINING, exercise=Exercise.CARD, + handoff_to_dds=handoff, scenario=operator, + criteria=LessonCriteria(require_correct_grammar=grammar), + ) + state.kio = KIO( + address="улица Ленина, 14", incident_type="fire", victims_count=2, + description="горит балкон", notify=["Служба 101"], + ) + state.on_event("card.start") + state.dispatch() + state.on_event("card.submit") + if handoff: + prepare_handoff_queue(state, [scenario("tickets/t01-1-fire-container.yaml")]) + return state + + +def dds_state() -> SessionState: + first = scenario("fire-apartment-l2.yaml") + state = SessionState( + session_id=uuid4(), scenario_id=first.id, scenario_title=first.title, + level=first.level.value, mode=SessionMode.TRAINING, exercise=Exercise.DDS, + criteria=LessonCriteria(require_correct_grammar=True), + ) + prepare_queue(state, [first, scenario("tickets/t20-2-stroke.yaml")]) + return state + + +def grammar_says(passed: bool, calls: list[str]): + async def check(text: str) -> GrammarAssessment: + calls.append(text) + return GrammarAssessment( + passed=passed, errors=() if passed else ("нет завершающего знака препинания",), + source="test", + ) + return check + + +def score(snapshot, grammar): + return asyncio.run(score_session(snapshot, grammar=grammar)) + + +@pytest.fixture(autouse=True) +def offline_finish(monkeypatch): + class NoCoaching: + def model_dump(self, **_kwargs): + return {} + + async def no_coach(_metrics): + return NoCoaching() + + monkeypatch.setattr(finish_module, "coach", no_coach) + monkeypatch.setattr(hub, "store", MemorySessionStore()) + + +def test_passed_grammar_check_adds_passed_metric_without_e4(): + calls: list[str] = [] + snapshot = ScoringSnapshot.of(card_state()) + + report = score(snapshot, grammar_says(True, calls)) + + assert calls == ["горит балкон"] + metric = next(item for item in report.result.metrics if item.key == "description_grammar") + assert metric.passed and metric.fact == "ошибок не обнаружено" + assert ErrorCode.E4 not in [item.code for item in report.result.findings] + + +def test_failed_grammar_check_adds_e4_and_lowers_score(): + passed = score(ScoringSnapshot.of(card_state()), grammar_says(True, [])) + failed = score(ScoringSnapshot.of(card_state()), grammar_says(False, [])) + + metric = next(item for item in failed.result.metrics if item.key == "description_grammar") + assert not metric.passed + assert metric.fact == "нет завершающего знака препинания" + e4 = next(item for item in failed.result.findings if item.code is ErrorCode.E4) + assert e4.source is FindingSource.GRAMMAR + assert failed.result.score < passed.result.score + + +def test_grammar_is_not_checked_without_criterion_or_on_dds_exercise(): + calls: list[str] = [] + + off = score(ScoringSnapshot.of(card_state(grammar=False)), grammar_says(False, calls)) + dds = score(ScoringSnapshot.of(dds_state()), grammar_says(False, calls)) + + assert calls == [] + assert "description_grammar" not in [item.key for item in off.result.metrics] + assert "description_grammar" not in [item.key for item in dds.result.metrics] + + +def test_snapshot_is_read_only(): + snapshot = ScoringSnapshot.of(card_state()) + + with pytest.raises(FrozenInstanceError): + snapshot.scenario = None + + +def test_card_fill_time_is_scored_from_snapshot_timers(): + report = score(ScoringSnapshot.of(card_state(grammar=False)), grammar_says(True, [])) + + metric = next(item for item in report.result.metrics if item.key == "card_fill_time") + assert metric.passed + assert metric.fact == "0 с (на 180 с быстрее норматива)" + assert report.summary["card_fill_ms"] is not None + + +def test_dds_snapshot_scores_every_queue_card_and_leaves_state_untouched(): + state = dds_state() + finish_module.score_live_dds_cards(state) + completed = list(state.desk.completed) + snapshot = ScoringSnapshot.of(state) + + report = score(snapshot, grammar_says(True, [])) + + assert [card.scenario_id for card in report.cards] == ["fire-apartment-l2", "t20-2-stroke"] + assert [item.title for item in report.result.metrics] == [ + item.title for card in completed for item in card.metrics + ] + assert state.desk.completed == completed + + +def test_handoff_live_and_refreshed_reports_use_operator_scenario_and_same_score(): + state = card_state(handoff=True) + active = state.desk.ordered()[1] + state.desk.open(active.card_id) + active.set_service_status( + active.dispatched_card.notify[0], ServiceStatus.ACCEPTED, + "Основание: доклад.\nСведения: принято.", + ) + # Предусловие ошибки: к завершению активна не карточка курсанта. + assert state.desk.active.scenario.id == "t01-1-fire-container" + assert ScoringSnapshot.of(state).scenario.id == "fire-apartment-l2" + + async def live_then_refreshed(): + async with hub.operation(state.session_id): + await finish_module.finish(state.session_id, state) + live = dict(state.score["full_report"]) + state.self_assessed = True + async with hub.operation(state.session_id): + await finish_module.refresh_archived_report(state.session_id, state) + return live, state.score["full_report"] + + live, refreshed = asyncio.run(live_then_refreshed()) + + assert live["scenario_id"] == refreshed["scenario_id"] == "fire-apartment-l2" + assert live["score_auto"] == refreshed["score_auto"] == state.score["score_auto"] + assert live["metrics"] == refreshed["metrics"]