refactor: оценка занятия по неизменяемому снимку, грамматика параметром, тесты без SimpleNamespace
This commit is contained in:
parent
a11206527a
commit
9f2ef7691f
5 changed files with 506 additions and 249 deletions
|
|
@ -29,7 +29,8 @@ from app.scoring.export import to_csv, to_pdf
|
||||||
from app.scoring.report import build as build_report
|
from app.scoring.report import build as build_report
|
||||||
from app.session.access import can_access
|
from app.session.access import can_access
|
||||||
from app.session.checkpoint import load_state
|
from app.session.checkpoint import load_state
|
||||||
from app.session.finish import override_score, scoring_scenario
|
from app.session.finish import override_score
|
||||||
|
from app.session.score import scoring_scenario
|
||||||
from app.session.hub import hub
|
from app.session.hub import hub
|
||||||
from app.session.store import ScoreOverridden, apply_score_override
|
from app.session.store import ScoreOverridden, apply_score_override
|
||||||
from app.voice.recording import recording_path
|
from app.voice.recording import recording_path
|
||||||
|
|
|
||||||
|
|
@ -12,22 +12,18 @@ import logging
|
||||||
import time
|
import time
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from app.domain.events import CallEnded, CallEndReason, Exercise, Metric, ScoreReady, SessionEnded
|
from app.domain.events import CallEnded, CallEndReason, Exercise, ScoreReady, SessionEnded
|
||||||
from app.domain.statuses import ServiceStatus, current
|
from app.domain.statuses import ServiceStatus, current
|
||||||
from app.domain.taxonomy import Competency, ErrorCode, Finding, FindingSource
|
|
||||||
from app.domain.timers import TimerCode
|
from app.domain.timers import TimerCode
|
||||||
from app.scenarios import store
|
|
||||||
from app.scoring.ai_coach import coach
|
from app.scoring.ai_coach import coach
|
||||||
from app.scoring.card import evaluate_card
|
|
||||||
from app.scoring.competency import radar
|
|
||||||
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
||||||
from app.scoring.gost import GostResult, evaluate
|
from app.scoring.gost import GostResult
|
||||||
from app.scoring.grammar import assess
|
from app.scoring.grammar import assess
|
||||||
from app.scoring.report import build as build_report
|
from app.scoring.report import build as build_report
|
||||||
from app.scoring.taxonomy import METRIC_WEIGHTS
|
from app.scoring.timing import DDS_WORK_TIME, time_metric
|
||||||
from app.scoring.timing import time_credit
|
|
||||||
from app.scoring.weights import apply_weights
|
from app.scoring.weights import apply_weights
|
||||||
from app.session.hub import hub
|
from app.session.hub import hub
|
||||||
|
from app.session.score import ScoringSnapshot, score_session, scoring_scenario
|
||||||
from app.session.state import DdsCardRecord, DdsLiveCard, now_utc
|
from app.session.state import DdsCardRecord, DdsLiveCard, now_utc
|
||||||
from app.session.store import (
|
from app.session.store import (
|
||||||
LessonEnded,
|
LessonEnded,
|
||||||
|
|
@ -77,39 +73,12 @@ def score_dds_card(state, card: DdsLiveCard) -> DdsCardRecord:
|
||||||
}
|
}
|
||||||
for service in services
|
for service in services
|
||||||
)
|
)
|
||||||
work_passed = terminal and work_elapsed_ms is not None and work_elapsed_ms <= work_limit_ms
|
work_metric, work_finding = time_metric(
|
||||||
work_delta_ms = (work_elapsed_ms - work_limit_ms) if work_elapsed_ms is not None else None
|
DDS_WORK_TIME, work_elapsed_ms, work_limit_ms, completed=terminal
|
||||||
if work_elapsed_ms is None:
|
)
|
||||||
work_fact = "время обработки не зафиксировано"
|
weighted.metrics.append(work_metric)
|
||||||
elif work_delta_ms and work_delta_ms > 0:
|
if work_finding is not None:
|
||||||
work_fact = f"{round(work_elapsed_ms / 1000)} с (+{round(work_delta_ms / 1000)} с сверх норматива)"
|
weighted.findings.append(work_finding)
|
||||||
elif work_delta_ms and work_delta_ms < 0:
|
|
||||||
work_fact = (f"{round(work_elapsed_ms / 1000)} с (на "
|
|
||||||
f"{round(abs(work_delta_ms) / 1000)} с быстрее норматива)")
|
|
||||||
else:
|
|
||||||
work_fact = f"{round(work_elapsed_ms / 1000)} с (точно в норматив)"
|
|
||||||
if not terminal:
|
|
||||||
work_fact = f"карточка не завершена; {work_fact}"
|
|
||||||
weighted.metrics.append(Metric(
|
|
||||||
key="dds_work_time",
|
|
||||||
title="Отработка карточки ДДС",
|
|
||||||
fact=work_fact,
|
|
||||||
norm=f"завершить за {round(work_limit_ms / 1000)} с",
|
|
||||||
ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026",
|
|
||||||
passed=work_passed,
|
|
||||||
weight=METRIC_WEIGHTS["dds_work_time"],
|
|
||||||
credit=time_credit(work_elapsed_ms, work_limit_ms) if terminal else 0.0,
|
|
||||||
))
|
|
||||||
if not work_passed:
|
|
||||||
weighted.findings.append(Finding(
|
|
||||||
code=ErrorCode.E3,
|
|
||||||
source=FindingSource.TIMERS,
|
|
||||||
summary="ДДС: норматив времени отработки карточки не выполнен",
|
|
||||||
fact=work_fact,
|
|
||||||
norm=f"завершить за {round(work_limit_ms / 1000)} с",
|
|
||||||
ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026",
|
|
||||||
competency=Competency.NORMS,
|
|
||||||
))
|
|
||||||
apply_weights(weighted, {**card.scenario.score_weights, **state.criteria.score_weights})
|
apply_weights(weighted, {**card.scenario.score_weights, **state.criteria.score_weights})
|
||||||
actions = [
|
actions = [
|
||||||
{"type": "card.status", "service": mark.service, "status": mark.status.value,
|
{"type": "card.status", "service": mark.service, "status": mark.status.value,
|
||||||
|
|
@ -149,20 +118,6 @@ def score_dds_card(state, card: DdsLiveCard) -> DdsCardRecord:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def scoring_scenario(state):
|
|
||||||
"""Сценарий, по которому оценивается и разбирается занятие.
|
|
||||||
|
|
||||||
После передачи 112→ДДС КИО курсанта оценивается по эталону упражнения
|
|
||||||
112. Балл, разбор и чек-лист самооценки обязаны брать один и тот же
|
|
||||||
сценарий.
|
|
||||||
"""
|
|
||||||
if state.handoff_to_dds and state.operator_scenario:
|
|
||||||
return state.operator_scenario
|
|
||||||
if state.scenario is None and state.desk.scenarios:
|
|
||||||
return state.desk.scenarios[0]
|
|
||||||
return state.scenario or store.get(state.scenario_id)
|
|
||||||
|
|
||||||
|
|
||||||
def score_live_dds_cards(state) -> list[DdsCardRecord]:
|
def score_live_dds_cards(state) -> list[DdsCardRecord]:
|
||||||
"""Все карточки занятия: завершённые и оценка ещё открытых.
|
"""Все карточки занятия: завершённые и оценка ещё открытых.
|
||||||
|
|
||||||
|
|
@ -182,185 +137,30 @@ async def finish(session_id: UUID, state) -> None:
|
||||||
if state.recorder is not None:
|
if state.recorder is not None:
|
||||||
path = await asyncio.to_thread(state.recorder.finalize)
|
path = await asyncio.to_thread(state.recorder.finalize)
|
||||||
state.recording_path = str(path) if path else None
|
state.recording_path = str(path) if path else None
|
||||||
|
# Все карточки выданы одновременно: при досрочном завершении оцениваем
|
||||||
|
# каждую, включая не открытую, потому что её норматив уже шёл.
|
||||||
|
if state.exercise is Exercise.DDS or (state.handoff_to_dds and state.desk.scenarios):
|
||||||
|
score_live_dds_cards(state)
|
||||||
# Сценарий занятия, а не библиотечный: директивы могли поправить эталон.
|
# Сценарий занятия, а не библиотечный: директивы могли поправить эталон.
|
||||||
scenario = scoring_scenario(state)
|
snapshot = ScoringSnapshot.of(state)
|
||||||
if scenario is None:
|
if snapshot is None:
|
||||||
return
|
return
|
||||||
|
report = await score_session(snapshot, grammar=assess)
|
||||||
cards: list[DdsCardRecord] = []
|
state.score = report.payload(card_status=state.station_snapshot().card.value)
|
||||||
if state.exercise is Exercise.CARD:
|
|
||||||
result = evaluate_card(
|
|
||||||
scenario, state.operator_kio or state.dispatched_card or state.kio
|
|
||||||
)
|
|
||||||
limit_ms = state.timers.limits[TimerCode.CARD_FILL]
|
|
||||||
elapsed_ms = state.timers.measured_ms(TimerCode.CARD_FILL)
|
|
||||||
submitted = state.dispatched_card is not None
|
|
||||||
if elapsed_ms is None:
|
|
||||||
elapsed_fact = "время не зафиксировано"
|
|
||||||
else:
|
|
||||||
delta_ms = elapsed_ms - limit_ms
|
|
||||||
elapsed_seconds = round(elapsed_ms / 1000)
|
|
||||||
if delta_ms > 0:
|
|
||||||
deviation = f"+{round(delta_ms / 1000)} с сверх норматива"
|
|
||||||
elif delta_ms < 0:
|
|
||||||
deviation = f"на {round(abs(delta_ms) / 1000)} с быстрее норматива"
|
|
||||||
else:
|
|
||||||
deviation = "точно в норматив"
|
|
||||||
elapsed_fact = f"{elapsed_seconds} с ({deviation})"
|
|
||||||
passed = submitted and elapsed_ms is not None and elapsed_ms <= limit_ms
|
|
||||||
result.metrics.append(Metric(
|
|
||||||
key="card_fill_time",
|
|
||||||
title="Время заполнения карточки",
|
|
||||||
fact=elapsed_fact if submitted else f"карточка не сдана; {elapsed_fact}",
|
|
||||||
norm=f"сдать карточку за {round(limit_ms / 1000)} с",
|
|
||||||
ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026",
|
|
||||||
passed=passed,
|
|
||||||
weight=METRIC_WEIGHTS["card_fill_time"],
|
|
||||||
credit=time_credit(elapsed_ms, limit_ms) if submitted else 0.0,
|
|
||||||
))
|
|
||||||
if not passed:
|
|
||||||
result.findings.append(Finding(
|
|
||||||
code=ErrorCode.E3,
|
|
||||||
source=FindingSource.TIMERS,
|
|
||||||
summary="Время заполнения карточки: норматив не выполнен",
|
|
||||||
fact=elapsed_fact if submitted else f"карточка не сдана; {elapsed_fact}",
|
|
||||||
norm=f"сдать карточку за {round(limit_ms / 1000)} с",
|
|
||||||
ref="ТЗ задачи № 9; сессия вопросов и ответов 16.09.2026",
|
|
||||||
competency=Competency.NORMS,
|
|
||||||
))
|
|
||||||
elif state.exercise is Exercise.DDS:
|
|
||||||
# Все карточки выданы одновременно: при досрочном завершении оцениваем
|
|
||||||
# каждую, включая не открытую, потому что её норматив уже шёл.
|
|
||||||
result = GostResult()
|
|
||||||
cards = score_live_dds_cards(state)
|
|
||||||
for card in cards:
|
|
||||||
result.metrics.extend(card.metrics)
|
|
||||||
result.findings.extend(card.findings)
|
|
||||||
else:
|
|
||||||
result = evaluate(
|
|
||||||
scenario=scenario,
|
|
||||||
kio=state.kio,
|
|
||||||
timers=state.timers,
|
|
||||||
revealed_facts=[fact.id for fact in state.slots.revealed_facts()] if state.slots else None,
|
|
||||||
refined_facts=list(state.slots.refined) if state.slots else None,
|
|
||||||
end_reason=state.end_reason,
|
|
||||||
bounced_fields=state.bounced_fields,
|
|
||||||
resolved_outcome=state.resolved_outcome,
|
|
||||||
dispatched=state.dispatched_card is not None,
|
|
||||||
)
|
|
||||||
# Грамматика относится к свободному описанию оператора 112, а не к
|
|
||||||
# заполнению карточки на стороне ДДС. Учитывается только по явной настройке
|
|
||||||
# преподавателя и только когда есть текст для проверки.
|
|
||||||
operator_kio = state.operator_kio if state.handoff_to_dds else None
|
|
||||||
description = (operator_kio or state.kio).description or ""
|
|
||||||
if (state.exercise is not Exercise.DDS and state.criteria.require_correct_grammar
|
|
||||||
and description.strip()):
|
|
||||||
grammar = await assess(description)
|
|
||||||
result.metrics.append(Metric(
|
|
||||||
key="description_grammar",
|
|
||||||
title="Грамматика описания происшествия",
|
|
||||||
fact="ошибок не обнаружено" if grammar.passed else "; ".join(grammar.errors),
|
|
||||||
norm="грамматически корректное описание",
|
|
||||||
ref="критерий занятия; правила русского языка",
|
|
||||||
passed=grammar.passed,
|
|
||||||
weight=1.0,
|
|
||||||
))
|
|
||||||
if not grammar.passed:
|
|
||||||
result.findings.append(Finding(
|
|
||||||
code=ErrorCode.E4,
|
|
||||||
source=FindingSource.GRAMMAR,
|
|
||||||
summary="Грамматическая ошибка в описании происшествия",
|
|
||||||
fact="; ".join(grammar.errors),
|
|
||||||
norm="грамматически корректное описание",
|
|
||||||
ref="критерий занятия; правила русского языка",
|
|
||||||
competency=Competency.COMMUNICATION,
|
|
||||||
))
|
|
||||||
# Работа диспетчера — вторая роль и вторая таксономия. Отметки D1–D6 идут
|
|
||||||
# рядом с E1–E6, а не вместо: в живой цепочке 112 → ДДС в одном занятии
|
|
||||||
# участвуют оба (docs/spec/DATASET.md#статусы-реагирования).
|
|
||||||
live_card = state.desk.active
|
|
||||||
if live_card is not None and state.exercise is Exercise.CALL:
|
|
||||||
decision_limit_ms = live_card.timers.limits[TimerCode.DDS_ACK]
|
|
||||||
services = state.card_services(live_card)
|
|
||||||
dispatcher_findings = evaluate_dispatcher(
|
|
||||||
entries=live_card.status_log,
|
|
||||||
services=services,
|
|
||||||
crew_assignments=live_card.crew_assignments,
|
|
||||||
deadline_ms=decision_limit_ms,
|
|
||||||
elapsed_ms=live_card.timers.measured_ms(TimerCode.DDS_ACK),
|
|
||||||
reply_text=live_card.reply_text,
|
|
||||||
expected_decision=scenario.dds_decision.expected,
|
|
||||||
expected_decision_reason=scenario.dds_decision.reason,
|
|
||||||
)
|
|
||||||
result.findings.extend(dispatcher_findings)
|
|
||||||
result.metrics.extend(dispatcher_metrics(
|
|
||||||
live_card, decision_limit_ms, scenario.dds_decision.expected,
|
|
||||||
scenario.dds_decision.reason, services=services,
|
|
||||||
))
|
|
||||||
# DDS cards were weighted individually in score_dds_card using each
|
|
||||||
# card's scenario defaults plus the lesson override. Reapplying the first
|
|
||||||
# scenario's weights here would corrupt the other ticket cards.
|
|
||||||
if state.exercise is not Exercise.DDS:
|
|
||||||
apply_weights(result, {**scenario.score_weights, **state.criteria.score_weights})
|
|
||||||
|
|
||||||
if state.handoff_to_dds and state.desk.scenarios:
|
|
||||||
# В связке КИО оценивается относительно эталона и весов упражнения
|
|
||||||
# 112, а каждая карточка очереди уже взвешена собственным сценарием.
|
|
||||||
cards = score_live_dds_cards(state)
|
|
||||||
for card in cards:
|
|
||||||
result.metrics.extend(card.metrics)
|
|
||||||
result.findings.extend(card.findings)
|
|
||||||
|
|
||||||
# Сводка числами: по ней считается дельта между попытками в профиле.
|
|
||||||
# Вытаскивать её разбором текста метрик («94 с») — путь к тихим ошибкам.
|
|
||||||
required = scenario.ground_truth.required_facts
|
|
||||||
revealed = [fact.id for fact in state.slots.revealed_facts()] if state.slots else []
|
|
||||||
codes: dict[str, int] = {}
|
|
||||||
for finding in result.findings:
|
|
||||||
codes[finding.code.value] = codes.get(finding.code.value, 0) + 1
|
|
||||||
|
|
||||||
state.score = {
|
|
||||||
"score_auto": result.score,
|
|
||||||
"summary": {
|
|
||||||
"interview_ms": state.timers.measured_ms(TimerCode.INTERVIEW),
|
|
||||||
"card_fill_ms": state.timers.measured_ms(TimerCode.CARD_FILL),
|
|
||||||
"facts_got": len([fact for fact in required if fact in revealed]),
|
|
||||||
"facts_required": len(required),
|
|
||||||
"hints": len(state.hints_shown),
|
|
||||||
"codes": codes,
|
|
||||||
},
|
|
||||||
"metrics": [metric.model_dump() for metric in result.metrics],
|
|
||||||
"findings": [finding.model_dump(mode="json") for finding in result.findings],
|
|
||||||
"competencies": [item.model_dump() for item in radar(result.metrics)],
|
|
||||||
"unavailable": result.unavailable,
|
|
||||||
# Статус карточки — готовая красная метка, понятная любому диспетчеру.
|
|
||||||
"card_status": state.station_snapshot().card.value,
|
|
||||||
"card_results": [
|
|
||||||
{"card_id": str(card.card_id), "scenario_id": card.scenario_id,
|
|
||||||
"score_auto": card.score_auto, "reply_text": card.reply_text,
|
|
||||||
"actions": card.actions, "duration_ms": card.duration_ms,
|
|
||||||
"title": card.title, "address": card.address,
|
|
||||||
"description": card.description, "incident_type": card.incident_type,
|
|
||||||
"victims_count": card.victims_count,
|
|
||||||
"received_at": card.received_at.isoformat() if card.received_at else None,
|
|
||||||
"managed_service": card.managed_service,
|
|
||||||
"recipient_services": card.recipient_services,
|
|
||||||
"metrics": [metric.model_dump(mode="json") for metric in card.metrics],
|
|
||||||
"findings": [finding.model_dump(mode="json") for finding in card.findings]}
|
|
||||||
for card in cards
|
|
||||||
] if state.exercise is Exercise.DDS or state.handoff_to_dds else [],
|
|
||||||
}
|
|
||||||
# Модель только поясняет уже посчитанные провалы и балл не трогает; без
|
# Модель только поясняет уже посчитанные провалы и балл не трогает; без
|
||||||
# запущенной модели разбор выходит со статусом «недоступно», а не ждёт её.
|
# запущенной модели разбор выходит со статусом «недоступно», а не ждёт её.
|
||||||
state.score["ai_coaching"] = (await coach(result.metrics)).model_dump(mode="json")
|
state.score["ai_coaching"] = (await coach(report.result.metrics)).model_dump(mode="json")
|
||||||
# Полный разбор хранится вместе с оценкой: PDF/CSV и история должны
|
# Полный разбор хранится вместе с оценкой: PDF/CSV и история должны
|
||||||
# переживать перезапуск backend, а не зависеть от живого объекта в реестре хаба.
|
# переживать перезапуск backend, а не зависеть от живого объекта в реестре хаба.
|
||||||
state.score["full_report"] = build_report(session_id, state, scenario).model_dump(mode="json")
|
state.score["full_report"] = build_report(
|
||||||
log.info("сессия %s: оценка %.1f, отметок %d", session_id, result.score, len(result.findings))
|
session_id, state, snapshot.scenario
|
||||||
|
).model_dump(mode="json")
|
||||||
|
log.info("сессия %s: оценка %.1f, отметок %d",
|
||||||
|
session_id, report.result.score, len(report.result.findings))
|
||||||
|
|
||||||
# Оценка и аудит уходят в commit операции, завершившей занятие; ScoreReady
|
# Оценка и аудит уходят в commit операции, завершившей занятие; ScoreReady
|
||||||
# рассылает `end_session` и ждёт того же commit — без записи итог не выдаётся.
|
# рассылает `end_session` и ждёт того же commit — без записи итог не выдаётся.
|
||||||
hub.record(session_id, ScoreCalculated(result.score, state.score))
|
hub.record(session_id, ScoreCalculated(report.result.score, state.score))
|
||||||
|
|
||||||
|
|
||||||
def _ends_on_station(state) -> bool:
|
def _ends_on_station(state) -> bool:
|
||||||
|
|
|
||||||
263
backend/app/session/score.py
Normal file
263
backend/app/session/score.py
Normal file
|
|
@ -0,0 +1,263 @@
|
||||||
|
"""Оценка занятия по неизменяемому снимку.
|
||||||
|
|
||||||
|
`finish()` сначала доводит живое занятие до оцениваемого вида (дооценивает
|
||||||
|
очередь ДДС), затем снимает `ScoringSnapshot` и отдаёт его `score_session`.
|
||||||
|
Сам расчёт не читает и не меняет `state`: живой, сохранённый и перестроенный
|
||||||
|
отчёт получают сценарий одним путём — через `scoring_scenario`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Awaitable, Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from app.domain.events import CallEndReason, Exercise, LessonCriteria, Metric
|
||||||
|
from app.domain.kio import KIO
|
||||||
|
from app.domain.taxonomy import Competency, ErrorCode, Finding, FindingSource
|
||||||
|
from app.domain.timers import TimerCode
|
||||||
|
from app.scenarios import store
|
||||||
|
from app.scenarios.schema import Scenario
|
||||||
|
from app.scoring.card import evaluate_card
|
||||||
|
from app.scoring.competency import radar
|
||||||
|
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
||||||
|
from app.scoring.gost import GostResult, evaluate
|
||||||
|
from app.scoring.grammar import GrammarAssessment, assess
|
||||||
|
from app.scoring.timing import CARD_FILL_TIME, time_metric
|
||||||
|
from app.scoring.weights import apply_weights
|
||||||
|
from app.session.dds import DdsCardRecord, DdsLiveCard
|
||||||
|
from app.session.timers import SessionTimers
|
||||||
|
|
||||||
|
GrammarCheck = Callable[[str], Awaitable[GrammarAssessment]]
|
||||||
|
|
||||||
|
|
||||||
|
def scoring_scenario(state) -> Scenario | None:
|
||||||
|
"""Сценарий, по которому оценивается и разбирается занятие.
|
||||||
|
|
||||||
|
После передачи 112→ДДС КИО курсанта оценивается по эталону упражнения
|
||||||
|
112. Балл, разбор и чек-лист самооценки обязаны брать один и тот же
|
||||||
|
сценарий.
|
||||||
|
"""
|
||||||
|
if state.handoff_to_dds and state.operator_scenario:
|
||||||
|
return state.operator_scenario
|
||||||
|
if state.scenario is None and state.desk.scenarios:
|
||||||
|
return state.desk.scenarios[0]
|
||||||
|
return state.scenario or store.get(state.scenario_id)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ScoringSnapshot:
|
||||||
|
"""Всё, что нужно для балла, снятое с занятия в момент завершения."""
|
||||||
|
|
||||||
|
scenario: Scenario
|
||||||
|
exercise: Exercise
|
||||||
|
handoff_to_dds: bool
|
||||||
|
#: КИО, которую сверяют с эталоном: в упражнении 112 — сданная карточка.
|
||||||
|
kio: KIO
|
||||||
|
#: Свободное описание оператора 112 — предмет проверки грамматики.
|
||||||
|
description: str
|
||||||
|
dispatched: bool
|
||||||
|
timers: SessionTimers
|
||||||
|
criteria: LessonCriteria
|
||||||
|
revealed_facts: tuple[str, ...] | None
|
||||||
|
refined_facts: tuple[str, ...] | None
|
||||||
|
end_reason: CallEndReason | None
|
||||||
|
bounced_fields: tuple[str, ...]
|
||||||
|
resolved_outcome: str | None
|
||||||
|
hints: int
|
||||||
|
#: Уже оценённые карточки очереди ДДС, в порядке выдачи.
|
||||||
|
dds_cards: tuple[DdsCardRecord, ...]
|
||||||
|
#: Карточка на пульте в упражнении по звонку: отметки D1–D6 рядом с E1–E6.
|
||||||
|
call_card: DdsLiveCard | None
|
||||||
|
call_card_services: tuple[str, ...]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def reports_cards(self) -> bool:
|
||||||
|
return self.exercise is Exercise.DDS or self.handoff_to_dds
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def of(cls, state) -> "ScoringSnapshot | None":
|
||||||
|
"""Снимок живого занятия; None — оценивать не по чему.
|
||||||
|
|
||||||
|
Очередь ДДС к этому моменту должна быть дооценена
|
||||||
|
(`finish.score_live_dds_cards`): снимок берёт готовые записи.
|
||||||
|
"""
|
||||||
|
scenario = scoring_scenario(state)
|
||||||
|
if scenario is None:
|
||||||
|
return None
|
||||||
|
if state.exercise is Exercise.CARD:
|
||||||
|
kio = state.operator_kio or state.dispatched_card or state.kio
|
||||||
|
else:
|
||||||
|
kio = state.kio
|
||||||
|
# Грамматика относится к свободному описанию оператора 112, а не к
|
||||||
|
# заполнению карточки на стороне ДДС.
|
||||||
|
operator_kio = state.operator_kio if state.handoff_to_dds else None
|
||||||
|
live_card = state.desk.active if state.exercise is Exercise.CALL else None
|
||||||
|
queue = state.exercise is Exercise.DDS or (state.handoff_to_dds and state.desk.scenarios)
|
||||||
|
return cls(
|
||||||
|
scenario=scenario,
|
||||||
|
exercise=state.exercise,
|
||||||
|
handoff_to_dds=state.handoff_to_dds,
|
||||||
|
kio=kio.model_copy(deep=True),
|
||||||
|
description=(operator_kio or state.kio).description or "",
|
||||||
|
dispatched=state.dispatched_card is not None,
|
||||||
|
timers=state.timers.model_copy(deep=True),
|
||||||
|
criteria=state.criteria.model_copy(deep=True),
|
||||||
|
revealed_facts=(tuple(fact.id for fact in state.slots.revealed_facts())
|
||||||
|
if state.slots else None),
|
||||||
|
refined_facts=tuple(state.slots.refined) if state.slots else None,
|
||||||
|
end_reason=state.end_reason,
|
||||||
|
bounced_fields=tuple(state.bounced_fields),
|
||||||
|
resolved_outcome=state.resolved_outcome,
|
||||||
|
hints=len(state.hints_shown),
|
||||||
|
dds_cards=tuple(state.desk.completed) if queue else (),
|
||||||
|
call_card=live_card.model_copy(deep=True) if live_card is not None else None,
|
||||||
|
call_card_services=(tuple(state.card_services(live_card))
|
||||||
|
if live_card is not None else ()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ScoreReport:
|
||||||
|
"""Посчитанная оценка: результат ГОСТ, сводка числами и карточки ДДС."""
|
||||||
|
|
||||||
|
result: GostResult
|
||||||
|
summary: dict[str, Any]
|
||||||
|
cards: tuple[DdsCardRecord, ...]
|
||||||
|
reports_cards: bool
|
||||||
|
|
||||||
|
def payload(self, *, card_status: str) -> dict[str, Any]:
|
||||||
|
"""Сохраняемая оценка занятия — без разбора модели и полного отчёта."""
|
||||||
|
return {
|
||||||
|
"score_auto": self.result.score,
|
||||||
|
"summary": self.summary,
|
||||||
|
"metrics": [metric.model_dump() for metric in self.result.metrics],
|
||||||
|
"findings": [finding.model_dump(mode="json") for finding in self.result.findings],
|
||||||
|
"competencies": [item.model_dump() for item in radar(self.result.metrics)],
|
||||||
|
"unavailable": self.result.unavailable,
|
||||||
|
# Статус карточки — готовая красная метка, понятная любому диспетчеру.
|
||||||
|
"card_status": card_status,
|
||||||
|
"card_results": [
|
||||||
|
{"card_id": str(card.card_id), "scenario_id": card.scenario_id,
|
||||||
|
"score_auto": card.score_auto, "reply_text": card.reply_text,
|
||||||
|
"actions": card.actions, "duration_ms": card.duration_ms,
|
||||||
|
"title": card.title, "address": card.address,
|
||||||
|
"description": card.description, "incident_type": card.incident_type,
|
||||||
|
"victims_count": card.victims_count,
|
||||||
|
"received_at": card.received_at.isoformat() if card.received_at else None,
|
||||||
|
"managed_service": card.managed_service,
|
||||||
|
"recipient_services": card.recipient_services,
|
||||||
|
"metrics": [metric.model_dump(mode="json") for metric in card.metrics],
|
||||||
|
"findings": [finding.model_dump(mode="json") for finding in card.findings]}
|
||||||
|
for card in self.cards
|
||||||
|
] if self.reports_cards else [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def score_session(
|
||||||
|
snapshot: ScoringSnapshot, *, grammar: GrammarCheck = assess
|
||||||
|
) -> ScoreReport:
|
||||||
|
"""Балл занятия. Единственный внешний вызов — проверка грамматики, и та
|
||||||
|
только по явному критерию преподавателя."""
|
||||||
|
scenario = snapshot.scenario
|
||||||
|
if snapshot.exercise is Exercise.CARD:
|
||||||
|
result = evaluate_card(scenario, snapshot.kio)
|
||||||
|
metric, finding = time_metric(
|
||||||
|
CARD_FILL_TIME,
|
||||||
|
snapshot.timers.measured_ms(TimerCode.CARD_FILL),
|
||||||
|
snapshot.timers.limits[TimerCode.CARD_FILL],
|
||||||
|
completed=snapshot.dispatched,
|
||||||
|
)
|
||||||
|
result.metrics.append(metric)
|
||||||
|
if finding is not None:
|
||||||
|
result.findings.append(finding)
|
||||||
|
elif snapshot.exercise is Exercise.DDS:
|
||||||
|
# Все карточки выданы одновременно: при досрочном завершении оценена
|
||||||
|
# каждая, включая не открытую, потому что её норматив уже шёл.
|
||||||
|
result = GostResult()
|
||||||
|
else:
|
||||||
|
result = evaluate(
|
||||||
|
scenario=scenario,
|
||||||
|
kio=snapshot.kio,
|
||||||
|
timers=snapshot.timers,
|
||||||
|
revealed_facts=(list(snapshot.revealed_facts)
|
||||||
|
if snapshot.revealed_facts is not None else None),
|
||||||
|
refined_facts=(list(snapshot.refined_facts)
|
||||||
|
if snapshot.refined_facts is not None else None),
|
||||||
|
end_reason=snapshot.end_reason,
|
||||||
|
bounced_fields=list(snapshot.bounced_fields),
|
||||||
|
resolved_outcome=snapshot.resolved_outcome,
|
||||||
|
dispatched=snapshot.dispatched,
|
||||||
|
)
|
||||||
|
# Учитывается только по явной настройке преподавателя и только когда есть
|
||||||
|
# текст для проверки.
|
||||||
|
if (snapshot.exercise is not Exercise.DDS and snapshot.criteria.require_correct_grammar
|
||||||
|
and snapshot.description.strip()):
|
||||||
|
verdict = await grammar(snapshot.description)
|
||||||
|
result.metrics.append(Metric(
|
||||||
|
key="description_grammar",
|
||||||
|
title="Грамматика описания происшествия",
|
||||||
|
fact="ошибок не обнаружено" if verdict.passed else "; ".join(verdict.errors),
|
||||||
|
norm="грамматически корректное описание",
|
||||||
|
ref="критерий занятия; правила русского языка",
|
||||||
|
passed=verdict.passed,
|
||||||
|
weight=1.0,
|
||||||
|
))
|
||||||
|
if not verdict.passed:
|
||||||
|
result.findings.append(Finding(
|
||||||
|
code=ErrorCode.E4,
|
||||||
|
source=FindingSource.GRAMMAR,
|
||||||
|
summary="Грамматическая ошибка в описании происшествия",
|
||||||
|
fact="; ".join(verdict.errors),
|
||||||
|
norm="грамматически корректное описание",
|
||||||
|
ref="критерий занятия; правила русского языка",
|
||||||
|
competency=Competency.COMMUNICATION,
|
||||||
|
))
|
||||||
|
# Работа диспетчера — вторая роль и вторая таксономия. Отметки D1–D6 идут
|
||||||
|
# рядом с E1–E6, а не вместо: в живой цепочке 112 → ДДС в одном занятии
|
||||||
|
# участвуют оба (docs/spec/DATASET.md#статусы-реагирования).
|
||||||
|
live_card = snapshot.call_card
|
||||||
|
if live_card is not None:
|
||||||
|
decision_limit_ms = live_card.timers.limits[TimerCode.DDS_ACK]
|
||||||
|
services = list(snapshot.call_card_services)
|
||||||
|
result.findings.extend(evaluate_dispatcher(
|
||||||
|
entries=live_card.status_log,
|
||||||
|
services=services,
|
||||||
|
crew_assignments=live_card.crew_assignments,
|
||||||
|
deadline_ms=decision_limit_ms,
|
||||||
|
elapsed_ms=live_card.timers.measured_ms(TimerCode.DDS_ACK),
|
||||||
|
reply_text=live_card.reply_text,
|
||||||
|
expected_decision=scenario.dds_decision.expected,
|
||||||
|
expected_decision_reason=scenario.dds_decision.reason,
|
||||||
|
))
|
||||||
|
result.metrics.extend(dispatcher_metrics(
|
||||||
|
live_card, decision_limit_ms, scenario.dds_decision.expected,
|
||||||
|
scenario.dds_decision.reason, services=services,
|
||||||
|
))
|
||||||
|
# Карточки ДДС уже взвешены каждая своим сценарием плюс настройкой
|
||||||
|
# занятия. Веса первого сценария поверх них испортили бы остальные.
|
||||||
|
if snapshot.exercise is not Exercise.DDS:
|
||||||
|
apply_weights(result, {**scenario.score_weights, **snapshot.criteria.score_weights})
|
||||||
|
# В связке КИО оценивается относительно эталона и весов упражнения 112,
|
||||||
|
# а каждая карточка очереди уже взвешена собственным сценарием.
|
||||||
|
for card in snapshot.dds_cards:
|
||||||
|
result.metrics.extend(card.metrics)
|
||||||
|
result.findings.extend(card.findings)
|
||||||
|
|
||||||
|
# Сводка числами: по ней считается дельта между попытками в профиле.
|
||||||
|
# Вытаскивать её разбором текста метрик («94 с») — путь к тихим ошибкам.
|
||||||
|
required = scenario.ground_truth.required_facts
|
||||||
|
revealed = snapshot.revealed_facts or ()
|
||||||
|
codes: dict[str, int] = {}
|
||||||
|
for finding in result.findings:
|
||||||
|
codes[finding.code.value] = codes.get(finding.code.value, 0) + 1
|
||||||
|
summary = {
|
||||||
|
"interview_ms": snapshot.timers.measured_ms(TimerCode.INTERVIEW),
|
||||||
|
"card_fill_ms": snapshot.timers.measured_ms(TimerCode.CARD_FILL),
|
||||||
|
"facts_got": len([fact for fact in required if fact in revealed]),
|
||||||
|
"facts_required": len(required),
|
||||||
|
"hints": snapshot.hints,
|
||||||
|
"codes": codes,
|
||||||
|
}
|
||||||
|
return ScoreReport(
|
||||||
|
result=result, summary=summary, cards=snapshot.dds_cards,
|
||||||
|
reports_cards=snapshot.reports_cards,
|
||||||
|
)
|
||||||
|
|
@ -1,12 +1,28 @@
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from types import SimpleNamespace
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
|
|
||||||
from app.domain.statuses import ServiceStatus, StatusEntry
|
from app.domain.statuses import ServiceStatus, StatusEntry
|
||||||
from app.domain.taxonomy import ErrorCode
|
from app.domain.taxonomy import ErrorCode
|
||||||
|
from app.scenarios.loader import load_file
|
||||||
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
||||||
|
from app.session.dds import DdsLiveCard, build_card
|
||||||
|
from app.session.timers import SessionTimers
|
||||||
|
|
||||||
|
LIBRARY = Path(__file__).resolve().parents[2] / "scenarios"
|
||||||
|
SERVICES = ["Служба 101"]
|
||||||
|
|
||||||
|
|
||||||
|
def live_card(*, status_log, crew_assignments, dispatched_at) -> DdsLiveCard:
|
||||||
|
"""Карточка пульта с заданным журналом статусов — вход оценки ДДС."""
|
||||||
|
scenario = load_file(LIBRARY / "fire-apartment-l2.yaml", LIBRARY)
|
||||||
|
card = build_card(scenario, 0, SessionTimers().limits)
|
||||||
|
card.status_log = status_log
|
||||||
|
card.crew_assignments = crew_assignments
|
||||||
|
card.dispatched_at = dispatched_at
|
||||||
|
return card
|
||||||
|
|
||||||
|
|
||||||
def test_late_primary_status_has_d1_finding_even_when_status_exists():
|
def test_late_primary_status_has_d1_finding_even_when_status_exists():
|
||||||
|
|
@ -110,8 +126,7 @@ def test_missing_manual_status_comment_has_a_d5_finding():
|
||||||
|
|
||||||
def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
||||||
at = datetime.now(timezone.utc)
|
at = datetime.now(timezone.utc)
|
||||||
state = SimpleNamespace(
|
card = live_card(
|
||||||
managed_services=lambda: ["Служба 101"],
|
|
||||||
status_log=[
|
status_log=[
|
||||||
StatusEntry(
|
StatusEntry(
|
||||||
service="Служба 101", status=ServiceStatus.ACCEPTED,
|
service="Служба 101", status=ServiceStatus.ACCEPTED,
|
||||||
|
|
@ -127,7 +142,7 @@ def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
||||||
)
|
)
|
||||||
|
|
||||||
reply = next(
|
reply = next(
|
||||||
metric for metric in dispatcher_metrics(state, 30_000)
|
metric for metric in dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||||
if metric.key == "dds_reply"
|
if metric.key == "dds_reply"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -135,9 +150,9 @@ def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
||||||
assert "к каждой ручной отметке" in reply.norm
|
assert "к каждой ручной отметке" in reply.norm
|
||||||
|
|
||||||
findings = evaluate_dispatcher(
|
findings = evaluate_dispatcher(
|
||||||
entries=state.status_log,
|
entries=card.status_log,
|
||||||
services=["Служба 101"],
|
services=["Служба 101"],
|
||||||
crew_assignments=state.crew_assignments,
|
crew_assignments=card.crew_assignments,
|
||||||
deadline_ms=30_000,
|
deadline_ms=30_000,
|
||||||
elapsed_ms=0,
|
elapsed_ms=0,
|
||||||
)
|
)
|
||||||
|
|
@ -173,13 +188,13 @@ def test_scenario_may_define_a_valid_decline_for_duplicate_or_territory():
|
||||||
expected_decision_reason="дублирующая карточка",
|
expected_decision_reason="дублирующая карточка",
|
||||||
)
|
)
|
||||||
assert ErrorCode.D3 not in [finding.code for finding in findings]
|
assert ErrorCode.D3 not in [finding.code for finding in findings]
|
||||||
state = SimpleNamespace(
|
card = live_card(
|
||||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
status_log=entries,
|
||||||
crew_assignments={}, dispatched_at=at,
|
crew_assignments={}, dispatched_at=at,
|
||||||
)
|
)
|
||||||
metrics = dispatcher_metrics(
|
metrics = dispatcher_metrics(
|
||||||
state, 30_000, expected_decision="decline",
|
card, 30_000, expected_decision="decline",
|
||||||
expected_decision_reason="дублирующая карточка",
|
expected_decision_reason="дублирующая карточка", services=SERVICES,
|
||||||
)
|
)
|
||||||
decision = next(metric for metric in metrics if metric.key == "dds_decision")
|
decision = next(metric for metric in metrics if metric.key == "dds_decision")
|
||||||
assert decision.passed
|
assert decision.passed
|
||||||
|
|
@ -188,8 +203,7 @@ def test_scenario_may_define_a_valid_decline_for_duplicate_or_territory():
|
||||||
|
|
||||||
def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes():
|
def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes():
|
||||||
at = datetime.now(timezone.utc)
|
at = datetime.now(timezone.utc)
|
||||||
state = SimpleNamespace(
|
card = live_card(
|
||||||
managed_services=lambda: ["Служба 101"],
|
|
||||||
status_log=[StatusEntry(
|
status_log=[StatusEntry(
|
||||||
service="Служба 101", status=ServiceStatus.ACCEPTED,
|
service="Служба 101", status=ServiceStatus.ACCEPTED,
|
||||||
at=at, comment="бригада на связи",
|
at=at, comment="бригада на связи",
|
||||||
|
|
@ -197,19 +211,19 @@ def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes(
|
||||||
crew_assignments={"Служба 101": "Бригада 12"},
|
crew_assignments={"Служба 101": "Бригада 12"},
|
||||||
dispatched_at=at,
|
dispatched_at=at,
|
||||||
)
|
)
|
||||||
reply = next(m for m in dispatcher_metrics(state, 30_000) if m.key == "dds_reply")
|
reply = next(m for m in dispatcher_metrics(card, 30_000, services=SERVICES) if m.key == "dds_reply")
|
||||||
assert not reply.passed
|
assert not reply.passed
|
||||||
findings = evaluate_dispatcher(
|
findings = evaluate_dispatcher(
|
||||||
entries=state.status_log, services=["Служба 101"],
|
entries=card.status_log, services=["Служба 101"],
|
||||||
crew_assignments=state.crew_assignments,
|
crew_assignments=card.crew_assignments,
|
||||||
deadline_ms=30_000, elapsed_ms=0,
|
deadline_ms=30_000, elapsed_ms=0,
|
||||||
)
|
)
|
||||||
assert any("не разделяют основание и сведения" in f.summary for f in findings)
|
assert any("не разделяют основание и сведения" in f.summary for f in findings)
|
||||||
|
|
||||||
state.status_log[0] = state.status_log[0].model_copy(update={
|
card.status_log[0] = card.status_log[0].model_copy(update={
|
||||||
"comment": "Основание: доклад старшего.\nСведения: бригада на связи.",
|
"comment": "Основание: доклад старшего.\nСведения: бригада на связи.",
|
||||||
})
|
})
|
||||||
reply = next(m for m in dispatcher_metrics(state, 30_000) if m.key == "dds_reply")
|
reply = next(m for m in dispatcher_metrics(card, 30_000, services=SERVICES) if m.key == "dds_reply")
|
||||||
assert reply.passed
|
assert reply.passed
|
||||||
|
|
||||||
def test_accepting_card_with_scenario_expected_decline_is_explained():
|
def test_accepting_card_with_scenario_expected_decline_is_explained():
|
||||||
|
|
@ -242,11 +256,11 @@ def test_incomplete_work_path_has_d6_for_failed_progress_metrics():
|
||||||
StatusEntry(service="Служба 101", status=ServiceStatus.ACCEPTED, at=at),
|
StatusEntry(service="Служба 101", status=ServiceStatus.ACCEPTED, at=at),
|
||||||
StatusEntry(service="Служба 101", status=ServiceStatus.RESPONDING, at=at),
|
StatusEntry(service="Служба 101", status=ServiceStatus.RESPONDING, at=at),
|
||||||
]
|
]
|
||||||
state = SimpleNamespace(
|
card = live_card(
|
||||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
status_log=entries,
|
||||||
crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at,
|
crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at,
|
||||||
)
|
)
|
||||||
metrics = dispatcher_metrics(state, 30_000)
|
metrics = dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||||
assert not next(item for item in metrics if item.key == "dds_progress").passed
|
assert not next(item for item in metrics if item.key == "dds_progress").passed
|
||||||
assert not next(item for item in metrics if item.key == "dds_completion").passed
|
assert not next(item for item in metrics if item.key == "dds_completion").passed
|
||||||
findings = evaluate_dispatcher(
|
findings = evaluate_dispatcher(
|
||||||
|
|
@ -267,11 +281,11 @@ def test_reasoned_refusal_after_acceptance_is_a_valid_terminal_path():
|
||||||
comment="Бригаде переданы сведения, выезд не выполнялся по причине угрозы.",
|
comment="Бригаде переданы сведения, выезд не выполнялся по причине угрозы.",
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
state = SimpleNamespace(
|
card = live_card(
|
||||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
status_log=entries,
|
||||||
crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at,
|
crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at,
|
||||||
)
|
)
|
||||||
metrics = dispatcher_metrics(state, 30_000)
|
metrics = dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||||
assert next(item for item in metrics if item.key == "dds_progress").passed
|
assert next(item for item in metrics if item.key == "dds_progress").passed
|
||||||
assert next(item for item in metrics if item.key == "dds_completion").passed
|
assert next(item for item in metrics if item.key == "dds_completion").passed
|
||||||
findings = evaluate_dispatcher(
|
findings = evaluate_dispatcher(
|
||||||
|
|
|
||||||
179
backend/tests/test_score_session.py
Normal file
179
backend/tests/test_score_session.py
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
"""Оценка занятия по снимку: без websocket, без живого `state` внутри расчёта."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from dataclasses import FrozenInstanceError
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.domain.events import Exercise, LessonCriteria, SessionMode
|
||||||
|
from app.domain.kio import KIO
|
||||||
|
from app.domain.statuses import ServiceStatus
|
||||||
|
from app.domain.taxonomy import ErrorCode, FindingSource
|
||||||
|
from app.scenarios.loader import load_file
|
||||||
|
from app.scoring.grammar import GrammarAssessment
|
||||||
|
from app.session import finish as finish_module
|
||||||
|
from app.session.dds import prepare_handoff_queue, prepare_queue
|
||||||
|
from app.session.hub import hub
|
||||||
|
from app.session.score import ScoringSnapshot, score_session
|
||||||
|
from app.session.state import SessionState
|
||||||
|
from app.session.store import MemorySessionStore
|
||||||
|
|
||||||
|
LIBRARY = Path(__file__).resolve().parents[2] / "scenarios"
|
||||||
|
|
||||||
|
|
||||||
|
def scenario(name: str):
|
||||||
|
return load_file(LIBRARY / name, LIBRARY)
|
||||||
|
|
||||||
|
|
||||||
|
def card_state(*, grammar: bool = True, handoff: bool = False) -> SessionState:
|
||||||
|
operator = scenario("fire-apartment-l2.yaml")
|
||||||
|
state = SessionState(
|
||||||
|
session_id=uuid4(), scenario_id=operator.id, scenario_title=operator.title,
|
||||||
|
level=operator.level.value, mode=SessionMode.TRAINING, exercise=Exercise.CARD,
|
||||||
|
handoff_to_dds=handoff, scenario=operator,
|
||||||
|
criteria=LessonCriteria(require_correct_grammar=grammar),
|
||||||
|
)
|
||||||
|
state.kio = KIO(
|
||||||
|
address="улица Ленина, 14", incident_type="fire", victims_count=2,
|
||||||
|
description="горит балкон", notify=["Служба 101"],
|
||||||
|
)
|
||||||
|
state.on_event("card.start")
|
||||||
|
state.dispatch()
|
||||||
|
state.on_event("card.submit")
|
||||||
|
if handoff:
|
||||||
|
prepare_handoff_queue(state, [scenario("tickets/t01-1-fire-container.yaml")])
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
def dds_state() -> SessionState:
|
||||||
|
first = scenario("fire-apartment-l2.yaml")
|
||||||
|
state = SessionState(
|
||||||
|
session_id=uuid4(), scenario_id=first.id, scenario_title=first.title,
|
||||||
|
level=first.level.value, mode=SessionMode.TRAINING, exercise=Exercise.DDS,
|
||||||
|
criteria=LessonCriteria(require_correct_grammar=True),
|
||||||
|
)
|
||||||
|
prepare_queue(state, [first, scenario("tickets/t20-2-stroke.yaml")])
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
def grammar_says(passed: bool, calls: list[str]):
|
||||||
|
async def check(text: str) -> GrammarAssessment:
|
||||||
|
calls.append(text)
|
||||||
|
return GrammarAssessment(
|
||||||
|
passed=passed, errors=() if passed else ("нет завершающего знака препинания",),
|
||||||
|
source="test",
|
||||||
|
)
|
||||||
|
return check
|
||||||
|
|
||||||
|
|
||||||
|
def score(snapshot, grammar):
|
||||||
|
return asyncio.run(score_session(snapshot, grammar=grammar))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def offline_finish(monkeypatch):
|
||||||
|
class NoCoaching:
|
||||||
|
def model_dump(self, **_kwargs):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
async def no_coach(_metrics):
|
||||||
|
return NoCoaching()
|
||||||
|
|
||||||
|
monkeypatch.setattr(finish_module, "coach", no_coach)
|
||||||
|
monkeypatch.setattr(hub, "store", MemorySessionStore())
|
||||||
|
|
||||||
|
|
||||||
|
def test_passed_grammar_check_adds_passed_metric_without_e4():
|
||||||
|
calls: list[str] = []
|
||||||
|
snapshot = ScoringSnapshot.of(card_state())
|
||||||
|
|
||||||
|
report = score(snapshot, grammar_says(True, calls))
|
||||||
|
|
||||||
|
assert calls == ["горит балкон"]
|
||||||
|
metric = next(item for item in report.result.metrics if item.key == "description_grammar")
|
||||||
|
assert metric.passed and metric.fact == "ошибок не обнаружено"
|
||||||
|
assert ErrorCode.E4 not in [item.code for item in report.result.findings]
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_grammar_check_adds_e4_and_lowers_score():
|
||||||
|
passed = score(ScoringSnapshot.of(card_state()), grammar_says(True, []))
|
||||||
|
failed = score(ScoringSnapshot.of(card_state()), grammar_says(False, []))
|
||||||
|
|
||||||
|
metric = next(item for item in failed.result.metrics if item.key == "description_grammar")
|
||||||
|
assert not metric.passed
|
||||||
|
assert metric.fact == "нет завершающего знака препинания"
|
||||||
|
e4 = next(item for item in failed.result.findings if item.code is ErrorCode.E4)
|
||||||
|
assert e4.source is FindingSource.GRAMMAR
|
||||||
|
assert failed.result.score < passed.result.score
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_is_not_checked_without_criterion_or_on_dds_exercise():
|
||||||
|
calls: list[str] = []
|
||||||
|
|
||||||
|
off = score(ScoringSnapshot.of(card_state(grammar=False)), grammar_says(False, calls))
|
||||||
|
dds = score(ScoringSnapshot.of(dds_state()), grammar_says(False, calls))
|
||||||
|
|
||||||
|
assert calls == []
|
||||||
|
assert "description_grammar" not in [item.key for item in off.result.metrics]
|
||||||
|
assert "description_grammar" not in [item.key for item in dds.result.metrics]
|
||||||
|
|
||||||
|
|
||||||
|
def test_snapshot_is_read_only():
|
||||||
|
snapshot = ScoringSnapshot.of(card_state())
|
||||||
|
|
||||||
|
with pytest.raises(FrozenInstanceError):
|
||||||
|
snapshot.scenario = None
|
||||||
|
|
||||||
|
|
||||||
|
def test_card_fill_time_is_scored_from_snapshot_timers():
|
||||||
|
report = score(ScoringSnapshot.of(card_state(grammar=False)), grammar_says(True, []))
|
||||||
|
|
||||||
|
metric = next(item for item in report.result.metrics if item.key == "card_fill_time")
|
||||||
|
assert metric.passed
|
||||||
|
assert metric.fact == "0 с (на 180 с быстрее норматива)"
|
||||||
|
assert report.summary["card_fill_ms"] is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_dds_snapshot_scores_every_queue_card_and_leaves_state_untouched():
|
||||||
|
state = dds_state()
|
||||||
|
finish_module.score_live_dds_cards(state)
|
||||||
|
completed = list(state.desk.completed)
|
||||||
|
snapshot = ScoringSnapshot.of(state)
|
||||||
|
|
||||||
|
report = score(snapshot, grammar_says(True, []))
|
||||||
|
|
||||||
|
assert [card.scenario_id for card in report.cards] == ["fire-apartment-l2", "t20-2-stroke"]
|
||||||
|
assert [item.title for item in report.result.metrics] == [
|
||||||
|
item.title for card in completed for item in card.metrics
|
||||||
|
]
|
||||||
|
assert state.desk.completed == completed
|
||||||
|
|
||||||
|
|
||||||
|
def test_handoff_live_and_refreshed_reports_use_operator_scenario_and_same_score():
|
||||||
|
state = card_state(handoff=True)
|
||||||
|
active = state.desk.ordered()[1]
|
||||||
|
state.desk.open(active.card_id)
|
||||||
|
active.set_service_status(
|
||||||
|
active.dispatched_card.notify[0], ServiceStatus.ACCEPTED,
|
||||||
|
"Основание: доклад.\nСведения: принято.",
|
||||||
|
)
|
||||||
|
# Предусловие ошибки: к завершению активна не карточка курсанта.
|
||||||
|
assert state.desk.active.scenario.id == "t01-1-fire-container"
|
||||||
|
assert ScoringSnapshot.of(state).scenario.id == "fire-apartment-l2"
|
||||||
|
|
||||||
|
async def live_then_refreshed():
|
||||||
|
async with hub.operation(state.session_id):
|
||||||
|
await finish_module.finish(state.session_id, state)
|
||||||
|
live = dict(state.score["full_report"])
|
||||||
|
state.self_assessed = True
|
||||||
|
async with hub.operation(state.session_id):
|
||||||
|
await finish_module.refresh_archived_report(state.session_id, state)
|
||||||
|
return live, state.score["full_report"]
|
||||||
|
|
||||||
|
live, refreshed = asyncio.run(live_then_refreshed())
|
||||||
|
|
||||||
|
assert live["scenario_id"] == refreshed["scenario_id"] == "fire-apartment-l2"
|
||||||
|
assert live["score_auto"] == refreshed["score_auto"] == state.score["score_auto"]
|
||||||
|
assert live["metrics"] == refreshed["metrics"]
|
||||||
Loading…
Reference in a new issue