refactor: оценка занятия по неизменяемому снимку, грамматика параметром, тесты без SimpleNamespace
This commit is contained in:
gglamer
parent
a11206527a
commit
9f2ef7691f
5 changed files with 506 additions and 249 deletions
|
|
@ -1,12 +1,28 @@
|
|||
from datetime import datetime, timezone
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.domain.statuses import ServiceStatus, StatusEntry
|
||||
from app.domain.taxonomy import ErrorCode
|
||||
from app.scenarios.loader import load_file
|
||||
from app.scoring.dispatcher import dispatcher_metrics, evaluate_dispatcher
|
||||
from app.session.dds import DdsLiveCard, build_card
|
||||
from app.session.timers import SessionTimers
|
||||
|
||||
LIBRARY = Path(__file__).resolve().parents[2] / "scenarios"
|
||||
SERVICES = ["Служба 101"]
|
||||
|
||||
|
||||
def live_card(*, status_log, crew_assignments, dispatched_at) -> DdsLiveCard:
|
||||
"""Карточка пульта с заданным журналом статусов — вход оценки ДДС."""
|
||||
scenario = load_file(LIBRARY / "fire-apartment-l2.yaml", LIBRARY)
|
||||
card = build_card(scenario, 0, SessionTimers().limits)
|
||||
card.status_log = status_log
|
||||
card.crew_assignments = crew_assignments
|
||||
card.dispatched_at = dispatched_at
|
||||
return card
|
||||
|
||||
|
||||
def test_late_primary_status_has_d1_finding_even_when_status_exists():
|
||||
|
|
@ -110,8 +126,7 @@ def test_missing_manual_status_comment_has_a_d5_finding():
|
|||
|
||||
def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
||||
at = datetime.now(timezone.utc)
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"],
|
||||
card = live_card(
|
||||
status_log=[
|
||||
StatusEntry(
|
||||
service="Служба 101", status=ServiceStatus.ACCEPTED,
|
||||
|
|
@ -127,7 +142,7 @@ def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
|||
)
|
||||
|
||||
reply = next(
|
||||
metric for metric in dispatcher_metrics(state, 30_000)
|
||||
metric for metric in dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||
if metric.key == "dds_reply"
|
||||
)
|
||||
|
||||
|
|
@ -135,9 +150,9 @@ def test_each_manual_status_comment_is_part_of_the_numeric_reply_metric():
|
|||
assert "к каждой ручной отметке" in reply.norm
|
||||
|
||||
findings = evaluate_dispatcher(
|
||||
entries=state.status_log,
|
||||
entries=card.status_log,
|
||||
services=["Служба 101"],
|
||||
crew_assignments=state.crew_assignments,
|
||||
crew_assignments=card.crew_assignments,
|
||||
deadline_ms=30_000,
|
||||
elapsed_ms=0,
|
||||
)
|
||||
|
|
@ -173,13 +188,13 @@ def test_scenario_may_define_a_valid_decline_for_duplicate_or_territory():
|
|||
expected_decision_reason="дублирующая карточка",
|
||||
)
|
||||
assert ErrorCode.D3 not in [finding.code for finding in findings]
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
||||
card = live_card(
|
||||
status_log=entries,
|
||||
crew_assignments={}, dispatched_at=at,
|
||||
)
|
||||
metrics = dispatcher_metrics(
|
||||
state, 30_000, expected_decision="decline",
|
||||
expected_decision_reason="дублирующая карточка",
|
||||
card, 30_000, expected_decision="decline",
|
||||
expected_decision_reason="дублирующая карточка", services=SERVICES,
|
||||
)
|
||||
decision = next(metric for metric in metrics if metric.key == "dds_decision")
|
||||
assert decision.passed
|
||||
|
|
@ -188,8 +203,7 @@ def test_scenario_may_define_a_valid_decline_for_duplicate_or_territory():
|
|||
|
||||
def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes():
|
||||
at = datetime.now(timezone.utc)
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"],
|
||||
card = live_card(
|
||||
status_log=[StatusEntry(
|
||||
service="Служба 101", status=ServiceStatus.ACCEPTED,
|
||||
at=at, comment="бригада на связи",
|
||||
|
|
@ -197,19 +211,19 @@ def test_nonempty_unstructured_comment_fails_reply_metric_but_structured_passes(
|
|||
crew_assignments={"Служба 101": "Бригада 12"},
|
||||
dispatched_at=at,
|
||||
)
|
||||
reply = next(m for m in dispatcher_metrics(state, 30_000) if m.key == "dds_reply")
|
||||
reply = next(m for m in dispatcher_metrics(card, 30_000, services=SERVICES) if m.key == "dds_reply")
|
||||
assert not reply.passed
|
||||
findings = evaluate_dispatcher(
|
||||
entries=state.status_log, services=["Служба 101"],
|
||||
crew_assignments=state.crew_assignments,
|
||||
entries=card.status_log, services=["Служба 101"],
|
||||
crew_assignments=card.crew_assignments,
|
||||
deadline_ms=30_000, elapsed_ms=0,
|
||||
)
|
||||
assert any("не разделяют основание и сведения" in f.summary for f in findings)
|
||||
|
||||
state.status_log[0] = state.status_log[0].model_copy(update={
|
||||
card.status_log[0] = card.status_log[0].model_copy(update={
|
||||
"comment": "Основание: доклад старшего.\nСведения: бригада на связи.",
|
||||
})
|
||||
reply = next(m for m in dispatcher_metrics(state, 30_000) if m.key == "dds_reply")
|
||||
reply = next(m for m in dispatcher_metrics(card, 30_000, services=SERVICES) if m.key == "dds_reply")
|
||||
assert reply.passed
|
||||
|
||||
def test_accepting_card_with_scenario_expected_decline_is_explained():
|
||||
|
|
@ -242,11 +256,11 @@ def test_incomplete_work_path_has_d6_for_failed_progress_metrics():
|
|||
StatusEntry(service="Служба 101", status=ServiceStatus.ACCEPTED, at=at),
|
||||
StatusEntry(service="Служба 101", status=ServiceStatus.RESPONDING, at=at),
|
||||
]
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
||||
card = live_card(
|
||||
status_log=entries,
|
||||
crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at,
|
||||
)
|
||||
metrics = dispatcher_metrics(state, 30_000)
|
||||
metrics = dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||
assert not next(item for item in metrics if item.key == "dds_progress").passed
|
||||
assert not next(item for item in metrics if item.key == "dds_completion").passed
|
||||
findings = evaluate_dispatcher(
|
||||
|
|
@ -267,11 +281,11 @@ def test_reasoned_refusal_after_acceptance_is_a_valid_terminal_path():
|
|||
comment="Бригаде переданы сведения, выезд не выполнялся по причине угрозы.",
|
||||
),
|
||||
]
|
||||
state = SimpleNamespace(
|
||||
managed_services=lambda: ["Служба 101"], status_log=entries,
|
||||
card = live_card(
|
||||
status_log=entries,
|
||||
crew_assignments={"Служба 101": "Бригада 1"}, dispatched_at=at,
|
||||
)
|
||||
metrics = dispatcher_metrics(state, 30_000)
|
||||
metrics = dispatcher_metrics(card, 30_000, services=SERVICES)
|
||||
assert next(item for item in metrics if item.key == "dds_progress").passed
|
||||
assert next(item for item in metrics if item.key == "dds_completion").passed
|
||||
findings = evaluate_dispatcher(
|
||||
|
|
|
|||
179
backend/tests/test_score_session.py
Normal file
179
backend/tests/test_score_session.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
"""Оценка занятия по снимку: без websocket, без живого `state` внутри расчёта."""
|
||||
|
||||
import asyncio
|
||||
from dataclasses import FrozenInstanceError
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.domain.events import Exercise, LessonCriteria, SessionMode
|
||||
from app.domain.kio import KIO
|
||||
from app.domain.statuses import ServiceStatus
|
||||
from app.domain.taxonomy import ErrorCode, FindingSource
|
||||
from app.scenarios.loader import load_file
|
||||
from app.scoring.grammar import GrammarAssessment
|
||||
from app.session import finish as finish_module
|
||||
from app.session.dds import prepare_handoff_queue, prepare_queue
|
||||
from app.session.hub import hub
|
||||
from app.session.score import ScoringSnapshot, score_session
|
||||
from app.session.state import SessionState
|
||||
from app.session.store import MemorySessionStore
|
||||
|
||||
LIBRARY = Path(__file__).resolve().parents[2] / "scenarios"
|
||||
|
||||
|
||||
def scenario(name: str):
|
||||
return load_file(LIBRARY / name, LIBRARY)
|
||||
|
||||
|
||||
def card_state(*, grammar: bool = True, handoff: bool = False) -> SessionState:
|
||||
operator = scenario("fire-apartment-l2.yaml")
|
||||
state = SessionState(
|
||||
session_id=uuid4(), scenario_id=operator.id, scenario_title=operator.title,
|
||||
level=operator.level.value, mode=SessionMode.TRAINING, exercise=Exercise.CARD,
|
||||
handoff_to_dds=handoff, scenario=operator,
|
||||
criteria=LessonCriteria(require_correct_grammar=grammar),
|
||||
)
|
||||
state.kio = KIO(
|
||||
address="улица Ленина, 14", incident_type="fire", victims_count=2,
|
||||
description="горит балкон", notify=["Служба 101"],
|
||||
)
|
||||
state.on_event("card.start")
|
||||
state.dispatch()
|
||||
state.on_event("card.submit")
|
||||
if handoff:
|
||||
prepare_handoff_queue(state, [scenario("tickets/t01-1-fire-container.yaml")])
|
||||
return state
|
||||
|
||||
|
||||
def dds_state() -> SessionState:
|
||||
first = scenario("fire-apartment-l2.yaml")
|
||||
state = SessionState(
|
||||
session_id=uuid4(), scenario_id=first.id, scenario_title=first.title,
|
||||
level=first.level.value, mode=SessionMode.TRAINING, exercise=Exercise.DDS,
|
||||
criteria=LessonCriteria(require_correct_grammar=True),
|
||||
)
|
||||
prepare_queue(state, [first, scenario("tickets/t20-2-stroke.yaml")])
|
||||
return state
|
||||
|
||||
|
||||
def grammar_says(passed: bool, calls: list[str]):
|
||||
async def check(text: str) -> GrammarAssessment:
|
||||
calls.append(text)
|
||||
return GrammarAssessment(
|
||||
passed=passed, errors=() if passed else ("нет завершающего знака препинания",),
|
||||
source="test",
|
||||
)
|
||||
return check
|
||||
|
||||
|
||||
def score(snapshot, grammar):
|
||||
return asyncio.run(score_session(snapshot, grammar=grammar))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def offline_finish(monkeypatch):
|
||||
class NoCoaching:
|
||||
def model_dump(self, **_kwargs):
|
||||
return {}
|
||||
|
||||
async def no_coach(_metrics):
|
||||
return NoCoaching()
|
||||
|
||||
monkeypatch.setattr(finish_module, "coach", no_coach)
|
||||
monkeypatch.setattr(hub, "store", MemorySessionStore())
|
||||
|
||||
|
||||
def test_passed_grammar_check_adds_passed_metric_without_e4():
|
||||
calls: list[str] = []
|
||||
snapshot = ScoringSnapshot.of(card_state())
|
||||
|
||||
report = score(snapshot, grammar_says(True, calls))
|
||||
|
||||
assert calls == ["горит балкон"]
|
||||
metric = next(item for item in report.result.metrics if item.key == "description_grammar")
|
||||
assert metric.passed and metric.fact == "ошибок не обнаружено"
|
||||
assert ErrorCode.E4 not in [item.code for item in report.result.findings]
|
||||
|
||||
|
||||
def test_failed_grammar_check_adds_e4_and_lowers_score():
|
||||
passed = score(ScoringSnapshot.of(card_state()), grammar_says(True, []))
|
||||
failed = score(ScoringSnapshot.of(card_state()), grammar_says(False, []))
|
||||
|
||||
metric = next(item for item in failed.result.metrics if item.key == "description_grammar")
|
||||
assert not metric.passed
|
||||
assert metric.fact == "нет завершающего знака препинания"
|
||||
e4 = next(item for item in failed.result.findings if item.code is ErrorCode.E4)
|
||||
assert e4.source is FindingSource.GRAMMAR
|
||||
assert failed.result.score < passed.result.score
|
||||
|
||||
|
||||
def test_grammar_is_not_checked_without_criterion_or_on_dds_exercise():
|
||||
calls: list[str] = []
|
||||
|
||||
off = score(ScoringSnapshot.of(card_state(grammar=False)), grammar_says(False, calls))
|
||||
dds = score(ScoringSnapshot.of(dds_state()), grammar_says(False, calls))
|
||||
|
||||
assert calls == []
|
||||
assert "description_grammar" not in [item.key for item in off.result.metrics]
|
||||
assert "description_grammar" not in [item.key for item in dds.result.metrics]
|
||||
|
||||
|
||||
def test_snapshot_is_read_only():
|
||||
snapshot = ScoringSnapshot.of(card_state())
|
||||
|
||||
with pytest.raises(FrozenInstanceError):
|
||||
snapshot.scenario = None
|
||||
|
||||
|
||||
def test_card_fill_time_is_scored_from_snapshot_timers():
|
||||
report = score(ScoringSnapshot.of(card_state(grammar=False)), grammar_says(True, []))
|
||||
|
||||
metric = next(item for item in report.result.metrics if item.key == "card_fill_time")
|
||||
assert metric.passed
|
||||
assert metric.fact == "0 с (на 180 с быстрее норматива)"
|
||||
assert report.summary["card_fill_ms"] is not None
|
||||
|
||||
|
||||
def test_dds_snapshot_scores_every_queue_card_and_leaves_state_untouched():
|
||||
state = dds_state()
|
||||
finish_module.score_live_dds_cards(state)
|
||||
completed = list(state.desk.completed)
|
||||
snapshot = ScoringSnapshot.of(state)
|
||||
|
||||
report = score(snapshot, grammar_says(True, []))
|
||||
|
||||
assert [card.scenario_id for card in report.cards] == ["fire-apartment-l2", "t20-2-stroke"]
|
||||
assert [item.title for item in report.result.metrics] == [
|
||||
item.title for card in completed for item in card.metrics
|
||||
]
|
||||
assert state.desk.completed == completed
|
||||
|
||||
|
||||
def test_handoff_live_and_refreshed_reports_use_operator_scenario_and_same_score():
|
||||
state = card_state(handoff=True)
|
||||
active = state.desk.ordered()[1]
|
||||
state.desk.open(active.card_id)
|
||||
active.set_service_status(
|
||||
active.dispatched_card.notify[0], ServiceStatus.ACCEPTED,
|
||||
"Основание: доклад.\nСведения: принято.",
|
||||
)
|
||||
# Предусловие ошибки: к завершению активна не карточка курсанта.
|
||||
assert state.desk.active.scenario.id == "t01-1-fire-container"
|
||||
assert ScoringSnapshot.of(state).scenario.id == "fire-apartment-l2"
|
||||
|
||||
async def live_then_refreshed():
|
||||
async with hub.operation(state.session_id):
|
||||
await finish_module.finish(state.session_id, state)
|
||||
live = dict(state.score["full_report"])
|
||||
state.self_assessed = True
|
||||
async with hub.operation(state.session_id):
|
||||
await finish_module.refresh_archived_report(state.session_id, state)
|
||||
return live, state.score["full_report"]
|
||||
|
||||
live, refreshed = asyncio.run(live_then_refreshed())
|
||||
|
||||
assert live["scenario_id"] == refreshed["scenario_id"] == "fire-apartment-l2"
|
||||
assert live["score_auto"] == refreshed["score_auto"] == state.score["score_auto"]
|
||||
assert live["metrics"] == refreshed["metrics"]
|
||||
Loading…
Reference in a new issue