pub/lct-hack
Watch
1
0
Fork
You've already forked lct-hack
 0
Code Releases Packages 5 Activity

Complete training workflow and acceptance hardening

Browse source
This commit is contained in:
andreysk0304 2026-09-26 18:12:27 +03:00 • committed by gglamer
commit 7237265833
243 changed files with 17014 additions and 1500 deletions
Download patch file Download diff file Expand all files Collapse all files

27
scripts/check_production_compose.sh Normal file
View file

@ -0,0 +1,27 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
compose=(docker compose -f "$repo_root/docker-compose.yml" \
-f "$repo_root/docker-compose.production.yml" -f "$repo_root/docker-compose.tls.yml")
if env -u POSTGRES_PASSWORD "${compose[@]}" config --quiet >/dev/null 2>&1; then
echo "ОШИБКА: production Compose запустился без POSTGRES_PASSWORD" >&2
exit 1
fi
secret="compose-check-0123456789abcdef0123456789abcdef"
case "$secret" in
*[!A-Za-z0-9._~-]*) echo "ОШИБКА: тестовый пароль не URL-safe" >&2; exit 1 ;;
esac
resolved="$(POSTGRES_PASSWORD="$secret" "${compose[@]}" config --format json 2>/dev/null)"
printf '%s' "$resolved" | jq -e --arg secret "$secret" '
.services.postgres.environment.POSTGRES_PASSWORD == $secret and
.services.backend.environment.DATABASE_URL == ("postgresql+asyncpg://lct:" + $secret + "@postgres:5432/lct") and
.services.backup.environment.DATABASE_URL == ("postgresql+asyncpg://lct:" + $secret + "@postgres:5432/lct") and
.services.backend.environment.APP_ENV == "production" and
.services.backend.environment.DEMO_NO_DB == "false" and
.services.backend.environment.DEV_AUTH_BYPASS == "false" and
.services.backend.environment.SECURE_COOKIES == "true"
' >/dev/null
echo "Production Compose требует уникальный пароль и передаёт его PostgreSQL/backend/backup."

698
scripts/load_browser.py
View file

@ -6,15 +6,196 @@ import asyncio
import inspect
import json
import math
import secrets
import ssl
import statistics
import subprocess
import sys
import time
import urllib.parse
import urllib.request
from uuid import uuid4
from pathlib import Path
from uuid import UUID, uuid4
import websockets
from playwright.async_api import async_playwright
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
async def create_distinct_users(
database_url: str, sessions: int, observers: int, *, create_controller: bool
) -> dict:
"""Provision short-lived local accounts for a literal 100-user load test."""
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.api.auth import hash_password
from app.db.models import Trainee, User
db_host = make_url(database_url).host
if db_host not in {"127.0.0.1", "localhost", "::1"}:
raise ValueError("--distinct-users разрешён только для loopback PostgreSQL")
suffix = uuid4().hex[:12]
engine = create_async_engine(database_url)
users: list[dict] = []
trainee_ids: list[str] = []
trainee_names: list[str] = []
controller = None
try:
factory = async_sessionmaker(engine, expire_on_commit=False)
async with factory() as db:
if create_controller:
controller = {
"login": f"load-i-{suffix}",
"password": secrets.token_urlsafe(24),
}
db.add(User(
login=controller["login"],
full_name=f"Нагрузочный преподаватель {suffix}",
password_hash=hash_password(controller["password"]),
role="instructor",
blocked=False,
))
for index in range(sessions):
name = f"Нагрузочный курсант {suffix}-{index:02d}"
login = f"load-t-{suffix}-{index:02d}"
password = secrets.token_urlsafe(24)
trainee = Trainee(name=name)
db.add(trainee)
await db.flush()
db.add(User(
login=login,
full_name=name,
password_hash=hash_password(password),
role="trainee",
trainee_id=trainee.id,
blocked=False,
))
users.append({"login": login, "password": password})
trainee_ids.append(str(trainee.id))
trainee_names.append(name)
for index in range(observers):
login = f"load-a-{suffix}-{index:03d}"
password = secrets.token_urlsafe(24)
db.add(User(
login=login,
full_name=f"Нагрузочный наблюдатель {suffix}-{index:03d}",
password_hash=hash_password(password),
role="admin",
blocked=False,
))
users.append({"login": login, "password": password})
await db.commit()
except Exception:
await engine.dispose()
raise
await engine.dispose()
return {
"suffix": suffix,
"users": users,
"controller": controller,
"trainee_ids": trainee_ids,
"trainee_names": trainee_names,
}
async def delete_distinct_users(database_url: str, fixtures: dict, session_ids: list[str]) -> None:
from sqlalchemy import delete
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.db.models import AuditLog, Session, Trainee, User
engine = create_async_engine(database_url)
try:
factory = async_sessionmaker(engine, expire_on_commit=False)
async with factory() as db:
ids = [UUID(value) for value in session_ids]
logins = [item["login"] for item in fixtures["users"]]
if fixtures["controller"]:
logins.append(fixtures["controller"]["login"])
if ids:
await db.execute(delete(Session).where(Session.id.in_(ids)))
await db.execute(delete(AuditLog).where(
AuditLog.object_id.in_([str(value) for value in ids])
))
await db.execute(delete(AuditLog).where(AuditLog.actor.in_(logins)))
await db.execute(delete(User).where(User.login.in_(logins)))
await db.execute(delete(Trainee).where(
Trainee.name.in_(fixtures["trainee_names"])
))
await db.commit()
finally:
await engine.dispose()
async def session_node_distribution(database_url: str, session_ids: list[str]) -> dict[str, int]:
"""Read the durable node owner for this run's sessions before cleanup."""
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.db.models import Session
engine = create_async_engine(database_url)
try:
async with async_sessionmaker(engine)() as db:
rows = (await db.execute(
select(Session.backend_node_id, func.count())
.where(Session.id.in_([UUID(value) for value in session_ids]))
.group_by(Session.backend_node_id)
)).all()
return {node or "<unassigned>": count for node, count in rows}
finally:
await engine.dispose()
def kill_cluster_service(project: str, service: str) -> None:
"""SIGKILL one backend only in an explicitly named disposable Compose project."""
if not project or not project.startswith("lct-"):
raise ValueError("fault injection requires an explicit lct-* disposable Compose project")
listing = subprocess.run(
["docker", "ps", "--filter", f"label=com.docker.compose.project={project}",
"--filter", f"label=com.docker.compose.service={service}", "--format", "{{.ID}}"],
check=False, capture_output=True, text=True,
)
containers = [line.strip() for line in listing.stdout.splitlines() if line.strip()]
if listing.returncode or len(containers) != 1:
raise RuntimeError(
f"expected exactly one running {service} in disposable project {project}; "
f"found {len(containers)}"
)
completed = subprocess.run(
["docker", "kill", containers[0]], check=False, capture_output=True, text=True,
)
if completed.returncode:
raise RuntimeError(f"Docker SIGKILL failed: {completed.stderr.strip()}")
async def wait_for_takeover(
database_url: str, session_ids: list[str], *, failed_node: str, timeout: float,
) -> tuple[dict[str, int], float]:
started = time.perf_counter()
deadline = started + timeout
distribution: dict[str, int] = {}
while time.perf_counter() < deadline:
distribution = await session_node_distribution(database_url, session_ids)
if sum(distribution.values()) == len(session_ids) and failed_node not in distribution:
return distribution, (time.perf_counter() - started) * 1000
await asyncio.sleep(0.5)
return distribution, (time.perf_counter() - started) * 1000
async def login_distinct_users(base: str, users: list[dict]) -> list[str]:
semaphore = asyncio.Semaphore(8)
async def login(user: dict) -> str:
async with semaphore:
return await asyncio.to_thread(login_cookie, base, user["login"], user["password"])
return await asyncio.gather(*(login(user) for user in users))
def local_url(value: str, schemes: set[str]) -> str:
parsed = urllib.parse.urlsplit(value)
@ -52,63 +233,256 @@ def login_cookie(base: str, login: str, password: str) -> str:
return value
async def control(ws_base: str, session_id: str, cookie: str, payload: dict) -> None:
async def control(
ws_base: str, session_id: str, cookie: str, payload: dict, *, ignore_https_errors: bool = False
) -> None:
header_arg = ("additional_headers" if "additional_headers" in inspect.signature(
websockets.connect
).parameters else "extra_headers")
options = {header_arg: {"Cookie": cookie}, "open_timeout": 10, "ping_interval": None}
if ws_base.startswith("wss://") and ignore_https_errors:
options["ssl"] = ssl._create_unverified_context()
async with websockets.connect(
f"{ws_base}/ws/control/{session_id}",
**{header_arg: {"Cookie": cookie}}, open_timeout=10, ping_interval=None,
**options,
) as socket:
await socket.send(json.dumps(payload, ensure_ascii=False))
await asyncio.sleep(0.03)
async def probe_call_ws(
ws_base: str, session_id: str, cookie: str, *, ignore_https_errors: bool,
) -> dict:
"""Probe a new call-channel handshake without competing for Chromium sockets."""
header_arg = ("additional_headers" if "additional_headers" in inspect.signature(
websockets.connect
).parameters else "extra_headers")
options = {
header_arg: {"Cookie": cookie}, "open_timeout": 5, "ping_interval": None,
}
if ws_base.startswith("wss://") and ignore_https_errors:
options["ssl"] = ssl._create_unverified_context()
started = time.perf_counter()
try:
async with websockets.connect(f"{ws_base}/ws/call/{session_id}", **options):
return {"ok": True, "elapsed_ms": (time.perf_counter() - started) * 1000}
except Exception as exc:
return {
"ok": False,
"elapsed_ms": (time.perf_counter() - started) * 1000,
"reason": f"{type(exc).__name__}: {str(exc)[:240]}",
}
def track_page_websockets(page, records: list[dict]) -> None:
"""Record only frame/close timestamps; never retain payloads or user data."""
def on_websocket(socket) -> None:
record = {
"url": socket.url,
"created_at": time.perf_counter(),
"frames": [],
"closed_at": None,
}
records.append(record)
def on_frame(frame, item=record) -> None:
event_type = None
error_code = None
if isinstance(frame, str):
try:
value = json.loads(frame)
if isinstance(value, dict):
event_type = value.get("type")
error_code = value.get("code")
except json.JSONDecodeError:
pass
item["frames"].append({
"at": time.perf_counter(), "type": event_type, "code": error_code,
})
socket.on("framereceived", on_frame)
socket.on("close", lambda _socket, item=record: item.update(
closed_at=time.perf_counter()
))
page.on("websocket", on_websocket)
def browser_channels_recovered(
records_by_page: list[list[dict]], expected_paths: list[str], after: float,
) -> tuple[list[int], list[float]]:
recovered: list[int] = []
first_frame_after: list[float] = []
for index, records in enumerate(records_by_page):
candidates = []
for record in records:
if expected_paths[index] not in record["url"]:
continue
post_failure_frames = [
frame["at"] for frame in record["frames"]
if frame["at"] >= after and frame["type"] is not None
and frame["type"] != "error" and frame["code"] is None
]
if (post_failure_frames
and (record["closed_at"] is None
or record["closed_at"] > post_failure_frames[-1])):
candidates.append(post_failure_frames[0])
if candidates:
recovered.append(index)
first_frame_after.append(min(candidates))
return recovered, first_frame_after
async def wait_for_browser_channels(
records_by_page: list[list[dict]], expected_paths: list[str], after: float, timeout: float,
) -> tuple[list[int], list[float]]:
deadline = time.perf_counter() + max(0, timeout)
while True:
recovered, first_frames = browser_channels_recovered(
records_by_page, expected_paths, after,
)
if len(recovered) == len(records_by_page) or time.perf_counter() >= deadline:
return recovered, first_frames
await asyncio.sleep(0.1)
async def run(args: argparse.Namespace) -> int:
frontend = local_url(args.frontend_url, {"http", "https"})
backend = local_url(args.backend_url, {"http"})
control_origin = local_url(args.control_url or args.backend_url, {"http", "https"})
if bool(args.login) != bool(args.password):
raise ValueError("--login и --password задаются вместе")
if not args.distinct_users and not args.login:
raise ValueError("обычный режим требует --login и --password")
if args.sessions < 1 or args.clients < args.sessions or args.clients % args.sessions:
raise ValueError("clients должно делиться на sessions и быть не меньше sessions")
if args.clients > 100 or args.sessions > 20:
raise ValueError("предохранитель: максимум 100 браузеров и 20 занятий")
if args.distinct_users and (args.sessions != 20 or args.clients != 100):
raise ValueError("--distinct-users проверяет ровно 20 занятий и 100 отдельных пользователей")
if args.ramp_seconds < 0:
raise ValueError("ramp-seconds не может быть отрицательным")
if not 0 <= args.expect_backend_nodes <= args.sessions:
raise ValueError("expect-backend-nodes должен быть от 0 до числа занятий")
if args.kill_backend_b_after_pages:
if not args.distinct_users or args.sessions != 20 or args.clients != 100:
raise ValueError("cluster fault-injection requires --distinct-users with 20 sessions / 100 clients")
if not args.database_url or not args.compose_project:
raise ValueError("cluster fault-injection requires --database-url and --compose-project")
if args.expect_backend_nodes < 2:
raise ValueError("fault-injection requires --expect-backend-nodes 2")
if control_origin != frontend or not frontend.startswith("https://"):
raise ValueError("fault-injection requires TLS Nginx for both frontend and control-url")
if args.failure_timeout <= 0:
raise ValueError("failure-timeout must be positive")
cookie = login_cookie(backend, args.login, args.password)
cookie_name, cookie_value = cookie.split("=", 1)
ws_base = "ws" + backend.removeprefix("http")
control_ws_base = control_origin.replace("https://", "wss://", 1).replace("http://", "ws://", 1)
sessions = [str(uuid4()) for _ in range(args.sessions)]
started: list[str] = []
fixtures = None
teacher_cookie = None
try:
await asyncio.gather(*(
control(ws_base, session_id, cookie, {
"type": "scenario.start", "scenario_id": args.scenario,
"trainee": f"browser-load-{index:02d}", "mode": "training", "exercise": "dds",
}) for index, session_id in enumerate(sessions)
))
station_cookies: list[str] = []
observer_cookies: list[str] = []
if args.distinct_users:
if not args.database_url:
raise ValueError("--distinct-users требует --database-url")
fixtures = await create_distinct_users(
args.database_url,
args.sessions,
args.clients - args.sessions,
create_controller=not bool(args.login),
)
controller = fixtures["controller"]
teacher_cookie = (
login_cookie(backend, controller["login"], controller["password"])
if controller
else login_cookie(backend, args.login, args.password)
)
account_cookies = await login_distinct_users(backend, fixtures["users"])
station_cookies = account_cookies[:args.sessions]
observer_cookies = account_cookies[args.sessions:]
else:
if not args.login or not args.password:
raise ValueError("обычный режим требует --login и --password")
teacher_cookie = login_cookie(backend, args.login, args.password)
# Include partially started sessions in cleanup if one concurrent
# control connection fails midway through the preparation batch.
started.extend(sessions)
await asyncio.gather(*(
control(control_ws_base, session_id, teacher_cookie, {
"type": "scenario.start", "scenario_id": args.scenario,
"trainee": (
fixtures["trainee_names"][index] if fixtures
else f"browser-load-{index:02d}"
),
"trainee_id": fixtures["trainee_ids"][index] if fixtures else None,
"mode": "training", "exercise": "dds",
}, ignore_https_errors=args.ignore_https_errors)
for index, session_id in enumerate(sessions)
))
timings: list[float] = []
dds_timings: list[float] = []
observer_timings: list[float] = []
browser_navigation: list[dict] = []
page_readiness_profiles: list[dict] = []
errors: list[str] = []
pages = []
contexts = []
per_session = args.clients // args.sessions
observer_index = 0
failure_injection = None
pre_failure_distribution: dict[str, int] = {}
websocket_records: list[list[dict]] = [[] for _ in range(args.clients)]
expected_ws_paths: list[str] = []
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
for index in range(args.clients):
session_index = index // per_session
role_index = index % per_session
if fixtures:
account_cookie = (
station_cookies[session_index]
if role_index == 0
else observer_cookies[observer_index]
)
if role_index != 0:
observer_index += 1
else:
account_cookie = teacher_cookie
cookie_name, cookie_value = account_cookie.split("=", 1)
context = await browser.new_context(
viewport={"width": 1280, "height": 720},
ignore_https_errors=args.ignore_https_errors,
)
await context.add_init_script("""
window.__lctLongTasks = [];
if (typeof PerformanceObserver !== "undefined") {
try {
new PerformanceObserver((list) => {
for (const item of list.getEntries()) {
window.__lctLongTasks.push({ start: item.startTime, duration: item.duration });
}
}).observe({ type: "longtask", buffered: true });
} catch (_) {}
}
""")
await context.add_cookies([{
"name": cookie_name, "value": cookie_value, "url": frontend,
"httpOnly": True, "sameSite": "Lax",
}])
page = await context.new_page()
page.on("pageerror", lambda error, i=index: errors.append(f"page {i}: {error}"))
session_index = index // per_session
role_index = index % per_session
session_id = sessions[session_index]
expected_ws_paths.append(
f"/ws/station/{session_id}" if role_index == 0
else f"/ws/observe/{session_id}"
)
track_page_websockets(page, websocket_records[index])
contexts.append(context)
pages.append(page)
@ -131,6 +505,7 @@ async def run(args: argparse.Namespace) -> int:
)
if response is None or not response.ok:
raise RuntimeError(f"HTTP {response.status if response else 'нет ответа'}")
document_ready_ms = (time.perf_counter() - started_at) * 1000
await pages[index].locator(selector).wait_for(timeout=args.timeout * 1000)
if role_index != 0:
await pages[index].wait_for_function(
@ -140,6 +515,68 @@ async def run(args: argparse.Namespace) -> int:
ready_ms = (time.perf_counter() - started_at) * 1000
timings.append(ready_ms)
(dds_timings if role_index == 0 else observer_timings).append(ready_ms)
profile = await pages[index].evaluate("""() => {
const resources = performance.getEntriesByType("resource")
.map((item) => {
const path = new URL(item.name).pathname.replace(
/[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}/gi, ":id"
);
return {
path,
initiator: item.initiatorType,
duration_ms: item.duration,
transfer_bytes: item.transferSize
};
})
.filter((item) => item.path.startsWith("/assets/")
|| item.path.startsWith("/api/"))
.sort((left, right) => right.duration_ms - left.duration_ms)
.slice(0, 30);
const tasks = window.__lctLongTasks ?? [];
return {
long_tasks: tasks.map((item) => ({
start_ms: item.start, duration_ms: item.duration
})),
resources
};
}""")
profile["websockets"] = []
for socket in websocket_records[index]:
socket_path = urllib.parse.urlsplit(socket["url"]).path
if socket_path.startswith("/ws/station/"):
socket_path = "/ws/station/:id"
elif socket_path.startswith("/ws/observe/"):
socket_path = "/ws/observe/:id"
elif socket_path.startswith("/ws/control/"):
socket_path = "/ws/control/:id"
frame_types = [frame["type"] for frame in socket["frames"]]
first_state = next((
frame for frame in socket["frames"]
if frame["type"] in {
"station.state", "observe.state", "session.snapshot",
"card.received",
}
), None)
profile["websockets"].append({
"path": socket_path,
"created_after_navigation_ms": round(
(socket["created_at"] - started_at) * 1000, 2
),
"first_frame_after_navigation_ms": round(
(socket["frames"][0]["at"] - started_at) * 1000, 2
) if socket["frames"] else None,
"first_state_after_navigation_ms": round(
(first_state["at"] - started_at) * 1000, 2
) if first_state else None,
"first_frame_type": frame_types[0] if frame_types else None,
})
page_readiness_profiles.append({
"page_index": index,
"role": "dds" if role_index == 0 else "wall",
"navigation_to_domcontentloaded_ms": round(document_ready_ms, 2),
"domcontentloaded_to_ready_ms": round(ready_ms - document_ready_ms, 2),
**profile,
})
navigation = await pages[index].evaluate("""() => {
const entry = performance.getEntriesByType('navigation')[0];
if (!entry) return null;
@ -157,14 +594,182 @@ async def run(args: argparse.Namespace) -> int:
errors.append(f"page {index} {path}: {type(exc).__name__}: {exc}")
await asyncio.gather(*(open_page(index) for index in range(args.clients)))
session_api_ok = 0
call_ws_ok = 0
route_errors: list[str] = []
if args.kill_backend_b_after_pages:
if len(timings) != args.clients or errors:
raise RuntimeError("all 100 pages must be ready before failure injection")
initial_ws_connected = sum(
any(
expected_ws_paths[index] in record["url"]
and any(
frame["type"] is not None
and frame["type"] != "error" and frame["code"] is None
for frame in record["frames"]
)
for record in records
)
for index, records in enumerate(websocket_records)
)
if initial_ws_connected != args.clients:
raise RuntimeError(
f"only {initial_ws_connected}/{args.clients} pages received their initial WS frame"
)
pre_failure_distribution = await session_node_distribution(
args.database_url, sessions,
)
if set(pre_failure_distribution) != {"backend-a", "backend-b"}:
raise RuntimeError(
"pre-failure sessions must be owned by both backend-a and backend-b"
)
failure_started = time.perf_counter()
kill_cluster_service(args.compose_project, "backend-b")
post_failure_distribution, takeover_ms = await wait_for_takeover(
args.database_url, sessions, failed_node="backend-b",
timeout=args.failure_timeout,
)
recovery_budget = max(
0.0,
min(30.0, args.failure_timeout) - (time.perf_counter() - failure_started),
)
recovered_pages, first_frames = await wait_for_browser_channels(
websocket_records, expected_ws_paths, failure_started, recovery_budget,
)
failure_injection = {
"action": "SIGKILL backend-b after all pages became ready",
"compose_project": args.compose_project,
"owner_distribution_before": pre_failure_distribution,
"owner_distribution_after": post_failure_distribution,
"takeover_ms": round(takeover_ms, 2),
"concurrent_pages_kept_open_during_takeover": args.clients,
"takeover_passed": (
sum(post_failure_distribution.values()) == args.sessions
and post_failure_distribution.get("backend-a") == args.sessions
and "backend-b" not in post_failure_distribution
),
"browser_ws_channel_pages_recovered": len(recovered_pages),
"browser_ws_channel_pages_expected": args.clients,
"browser_ws_initial_pages_connected": initial_ws_connected,
"browser_ws_unrecovered_pages": [
{
"page_index": index,
"channel": expected_ws_paths[index],
"session_id": sessions[index // per_session],
}
for index in range(args.clients) if index not in recovered_pages
],
"browser_ws_channels_recovered_within_30s": (
len(recovered_pages) == args.clients
and bool(first_frames)
and max(first_frames) - failure_started <= 30.0
),
"browser_ws_recovery_ms": (
round((max(first_frames) - failure_started) * 1000, 2)
if len(recovered_pages) == args.clients else None
),
}
failure_injection["elapsed_since_kill_ms"] = round(
(time.perf_counter() - failure_started) * 1000, 2
)
mounted = await asyncio.gather(*(
page.evaluate("Boolean(document.querySelector('#root')?.firstElementChild)")
for page in pages
))
failure_injection["react_pages_still_mounted"] = sum(bool(item) for item in mounted)
async def probe_session_api(session_index: int) -> dict:
page = pages[session_index * per_session]
return await page.evaluate("""async (sessionId) => {
const started = performance.now();
const response = await fetch(`/api/sessions/${sessionId}`, {
credentials: "same-origin"
});
return {
ok: response.status === 200,
status: response.status,
elapsed_ms: performance.now() - started
};
}""", sessions[session_index])
api_probes = await asyncio.gather(*(
probe_session_api(session_index) for session_index in range(args.sessions)
))
ws_probe_cookies = (
station_cookies if fixtures else [teacher_cookie] * args.sessions
)
ws_probes = await asyncio.gather(*(
probe_call_ws(
control_ws_base, sessions[index], ws_probe_cookies[index],
ignore_https_errors=args.ignore_https_errors,
)
for index in range(args.sessions)
))
if failure_injection:
failure_injection["route_probe_ms"] = {
"session_api_max": round(max(item["elapsed_ms"] for item in api_probes), 2),
"call_ws_max": round(max(item["elapsed_ms"] for item in ws_probes), 2),
"call_ws_probe_client": "Python websockets; Chromium pages remain open",
}
for session_index, (api_probe, ws_probe) in enumerate(zip(api_probes, ws_probes)):
if api_probe["ok"]:
session_api_ok += 1
else:
route_errors.append(
f"session {sessions[session_index]}: GET session returned {api_probe['status']}"
)
if ws_probe["ok"]:
call_ws_ok += 1
else:
route_errors.append(
f"session {sessions[session_index]}: /ws/call route failed: {ws_probe}"
)
await asyncio.sleep(args.hold_seconds)
await asyncio.gather(*(context.close() for context in contexts))
await browser.close()
node_distribution = (
await session_node_distribution(args.database_url, sessions)
if args.database_url else {}
)
nodes_for_expectation = pre_failure_distribution if failure_injection else node_distribution
expected_nodes_ready = (
len(nodes_for_expectation) >= args.expect_backend_nodes
if args.expect_backend_nodes else True
)
if failure_injection:
failure_injection["session_rest_routes_recovered"] = session_api_ok == args.sessions
failure_injection["call_ws_routes_recovered"] = call_ws_ok == args.sessions
failure_injection["route_errors_after_takeover"] = list(route_errors)
failure_injection["all_probes_recovered"] = (
session_api_ok == args.sessions and call_ws_ok == args.sessions
and not route_errors
)
failure_injection["pass_20_session_takeover"] = (
failure_injection["takeover_passed"]
and failure_injection["all_probes_recovered"]
and failure_injection["browser_ws_channels_recovered_within_30s"]
and failure_injection["react_pages_still_mounted"] == args.clients
)
result = {
"frontend": frontend, "backend": backend,
"frontend": frontend, "backend": backend, "control": control_origin,
"sessions_started": len(sessions), "browser_contexts": args.clients,
"dds_pages": args.sessions, "observer_pages": args.clients - args.sessions,
"distinct_authenticated_users": (
args.clients if fixtures else 1
),
"distinct_users_mode": bool(fixtures),
"backend_node_distribution": node_distribution,
"backend_node_count": len(node_distribution),
"backend_node_distribution_before_failure": (
pre_failure_distribution if failure_injection else None
),
"failure_injection": failure_injection,
"session_rest_routes_ok": session_api_ok,
"call_ws_routes_ok": call_ws_ok,
"route_errors": route_errors,
"pages_ready": len(timings), "hold_seconds": args.hold_seconds,
"ramp_seconds": args.ramp_seconds,
"ready_ms": timing_summary(timings),
@ -172,6 +777,14 @@ async def run(args: argparse.Namespace) -> int:
"dds": timing_summary(dds_timings),
"wall": timing_summary(observer_timings),
},
"page_readiness_profiles": page_readiness_profiles,
"pass_wall_report_deferred": not any(
path == "/api/sessions/:id/report"
or path.startswith("/assets/Debrief-")
or path.startswith("/assets/Radar-")
for profile in page_readiness_profiles if profile.get("role") == "wall"
for path in (resource.get("path", "") for resource in profile.get("resources", []))
),
"browser_navigation_ms": {
key: timing_summary([float(item[key]) for item in browser_navigation])
for key in ("responseStart", "responseEnd", "domInteractive", "domContentLoaded")
@ -181,13 +794,30 @@ async def run(args: argparse.Namespace) -> int:
]),
"errors": errors[:50],
"pass_browser_ui_2s": (
len(timings) == args.clients and not errors and percentile(timings, 95) <= 2000
len(timings) == args.clients
and not errors
and not route_errors
and session_api_ok == args.sessions
and call_ws_ok == args.sessions
and percentile(timings, 95) <= 2000
and len(dds_timings) == args.sessions
and len(observer_timings) == args.clients - args.sessions
and percentile(dds_timings, 95) <= 2000
and percentile(observer_timings, 95) <= 2000
and expected_nodes_ready
),
"pass_failover_under_20x100": bool(
failure_injection and failure_injection["pass_20_session_takeover"]
) if args.kill_backend_b_after_pages else None,
"scope": (
f"{args.clients} isolated Chromium contexts render real React DDS/wall pages "
f"for {args.sessions} sessions with a {args.ramp_seconds:g}s connection ramp; "
"all contexts remain active concurrently; same instructor account, local host, no VoIP, "
"all contexts remain active concurrently; "
+ ("unique trainee/admin accounts" if fixtures else "same instructor account")
+ ", local host, no VoIP, "
"not target 6-core/32-GB server"
+ (f"; at least {args.expect_backend_nodes} backend nodes required"
if args.expect_backend_nodes else "")
),
}
rendered = json.dumps(result, ensure_ascii=False, indent=2)
@ -195,20 +825,32 @@ async def run(args: argparse.Namespace) -> int:
if args.output:
with open(args.output, "w", encoding="utf-8") as stream:
stream.write(rendered + "\n")
return 0 if result["pass_browser_ui_2s"] else 1
passed = result["pass_browser_ui_2s"] and (
result["pass_wall_report_deferred"]
and result["pass_failover_under_20x100"] is not False
)
return 0 if passed else 1
finally:
await asyncio.gather(*(
control(ws_base, session_id, cookie, {"type": "session.stop"})
for session_id in started
), return_exceptions=True)
try:
if teacher_cookie and started:
await asyncio.gather(*(
control(control_ws_base, session_id, teacher_cookie, {"type": "session.stop"},
ignore_https_errors=args.ignore_https_errors)
for session_id in started
), return_exceptions=True)
finally:
if fixtures:
await delete_distinct_users(args.database_url, fixtures, started)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--frontend-url", default="http://127.0.0.1:5173")
parser.add_argument("--backend-url", default="http://127.0.0.1:8000")
parser.add_argument("--login", required=True)
parser.add_argument("--password", required=True)
parser.add_argument("--control-url",
help="URL для управляющего WS; задайте frontend URL для проверки proxy-affinity")
parser.add_argument("--login", help="преподаватель; в --distinct-users создаётся временный")
parser.add_argument("--password", help="пароль преподавателя, если задан --login")
parser.add_argument("--sessions", type=int, default=20)
parser.add_argument("--clients", type=int, default=100)
parser.add_argument("--scenario", default="fire-apartment-l2")
@ -216,6 +858,18 @@ if __name__ == "__main__":
parser.add_argument("--hold-seconds", type=float, default=2)
parser.add_argument("--ramp-seconds", type=float, default=0,
help="равномерно распределить начало загрузок; все страницы остаются активны")
parser.add_argument("--distinct-users", action="store_true",
help="создать/удалить временные учётки и проверить разные логины")
parser.add_argument("--database-url",
help="PostgreSQL URL, обязателен для --distinct-users; только тестовая БД")
parser.add_argument("--expect-backend-nodes", type=int, default=0,
help="проверить, что занятия распределены минимум по N backend-узлам")
parser.add_argument("--kill-backend-b-after-pages", action="store_true",
help="SIGKILL backend-b after all 100 pages load; isolated cluster only")
parser.add_argument("--compose-project",
help="required unique lct-* project for --kill-backend-b-after-pages")
parser.add_argument("--failure-timeout", type=float, default=45,
help="maximum seconds to wait for leases/checkpoints to transfer")
parser.add_argument("--output", help="сохранить JSON-отчёт")
parser.add_argument("--ignore-https-errors", action="store_true",
help="разрешить bootstrap self-signed сертификат только в локальном тесте")

137
scripts/load_db.py
View file

@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Проверка последовательной записи PostgreSQL без изменения данных продукта."""
"""Проверка локальной пропускной способности записи PostgreSQL."""
import argparse
import asyncio
@ -9,6 +9,7 @@ import os
import statistics
import time
import urllib.parse
from uuid import uuid4
import asyncpg
@ -31,36 +32,112 @@ def percentile(values: list[float], percent: float) -> float:
async def run(args: argparse.Namespace) -> int:
if not 100 <= args.operations <= 10_000:
raise ValueError("operations должно быть от 100 до 10000")
if not 1 <= args.concurrency <= 100:
raise ValueError("concurrency должно быть от 1 до 100")
if args.workload == "temporary" and args.concurrency != 1:
raise ValueError("temporary workload поддерживает только concurrency=1")
database_url = local_database_url(args.database_url)
connection = await asyncpg.connect(database_url, timeout=10)
worker_count = min(args.concurrency, args.operations)
pool = (
await asyncpg.create_pool(
database_url, min_size=worker_count, max_size=worker_count,
command_timeout=30,
)
if args.workload == "audit" and worker_count > 1 else None
)
connection = await asyncpg.connect(database_url, timeout=10) if pool is None else None
latencies: list[float] = []
run_marker = None
cleanup_rows = 0
try:
async with connection.transaction():
await connection.execute("""
CREATE TEMPORARY TABLE lct_write_benchmark (
sequence_no integer PRIMARY KEY,
written_at timestamptz NOT NULL DEFAULT now(),
payload text NOT NULL
) ON COMMIT DROP
""")
if args.workload == "temporary":
assert connection is not None
async with connection.transaction():
await connection.execute("""
CREATE TEMPORARY TABLE lct_write_benchmark (
sequence_no integer PRIMARY KEY,
written_at timestamptz NOT NULL DEFAULT now(),
payload text NOT NULL
) ON COMMIT DROP
""")
started = time.perf_counter()
for sequence_no in range(args.operations):
operation_started = time.perf_counter()
await connection.execute(
"INSERT INTO lct_write_benchmark(sequence_no, payload) VALUES($1, $2)",
sequence_no, f"local-load-{sequence_no:05d}",
)
latencies.append((time.perf_counter() - operation_started) * 1000)
elapsed = time.perf_counter() - started
count = await connection.fetchval("SELECT count(*) FROM lct_write_benchmark")
else:
if not args.allow_audit_writes:
raise ValueError("для workload=audit требуется явный --allow-audit-writes")
assert pool is not None or connection is not None
fetchval = pool.fetchval if pool is not None else connection.fetchval
if await fetchval("SELECT to_regclass('public.audit_log')") is None:
raise ValueError("таблица public.audit_log отсутствует; сначала примените миграции")
# This opt-in mode measures durable product-table writes. Every
# INSERT autocommits and exercises the real audit index; exact
# run-scoped rows are removed in finally, even after a partial run.
run_marker = f"load-benchmark:{uuid4().hex}"
started = time.perf_counter()
for sequence_no in range(args.operations):
operation_started = time.perf_counter()
await connection.execute(
"INSERT INTO lct_write_benchmark(sequence_no, payload) VALUES($1, $2)",
sequence_no, f"local-load-{sequence_no:05d}",
)
latencies.append((time.perf_counter() - operation_started) * 1000)
async def write_partition(worker_index: int) -> None:
if pool is None:
assert connection is not None
worker_connection = connection
for sequence_no in range(worker_index, args.operations, worker_count):
operation_started = time.perf_counter()
await worker_connection.execute(
"""INSERT INTO public.audit_log(id, actor, role, action, object_id, detail)
VALUES($1, $2, 'system', 'benchmark.write', $3, $4)""",
uuid4(), "load-benchmark", run_marker, f"sequence={sequence_no}",
)
latencies.append((time.perf_counter() - operation_started) * 1000)
else:
async with pool.acquire() as worker_connection:
for sequence_no in range(worker_index, args.operations, worker_count):
operation_started = time.perf_counter()
await worker_connection.execute(
"""INSERT INTO public.audit_log(id, actor, role, action, object_id, detail)
VALUES($1, $2, 'system', 'benchmark.write', $3, $4)""",
uuid4(), "load-benchmark", run_marker, f"sequence={sequence_no}",
)
latencies.append((time.perf_counter() - operation_started) * 1000)
await asyncio.gather(*(write_partition(worker) for worker in range(worker_count)))
elapsed = time.perf_counter() - started
count = await connection.fetchval("SELECT count(*) FROM lct_write_benchmark")
count = await fetchval(
"SELECT count(*) FROM public.audit_log WHERE object_id = $1", run_marker
)
finally:
await connection.close()
try:
if run_marker is not None:
if pool is not None:
cleanup_rows = await pool.fetchval(
"WITH removed AS (DELETE FROM public.audit_log WHERE object_id = $1 RETURNING 1) "
"SELECT count(*) FROM removed", run_marker,
)
elif connection is not None:
cleanup_rows = await connection.fetchval(
"WITH removed AS (DELETE FROM public.audit_log WHERE object_id = $1 RETURNING 1) "
"SELECT count(*) FROM removed", run_marker,
)
finally:
if pool is not None:
await pool.close()
elif connection is not None:
await connection.close()
throughput = args.operations / elapsed
result = {
"database": "local PostgreSQL",
"workload": args.workload,
"operations_requested": args.operations,
"operations_written": count,
"cleanup_rows_deleted": cleanup_rows,
"concurrency": worker_count,
"elapsed_seconds": elapsed,
"operations_per_second": throughput,
"operation_latency_ms": {
@ -69,9 +146,17 @@ async def run(args: argparse.Namespace) -> int:
"p95": percentile(latencies, 95),
"max": max(latencies),
},
"pass_100_writes_per_second": count == args.operations and throughput >= 100,
"pass_100_writes_per_second": (
count == args.operations
and throughput >= 100
and (args.workload != "audit" or cleanup_rows == args.operations)
),
"scope": (
"sequential INSERT statements over one local asyncpg connection into a temporary "
f"autocommitted INSERTs over {worker_count} local asyncpg connection(s) into the "
"indexed product audit_log table; this run's rows are deleted in finally; "
"not target 6-core/32-GB hardware"
if args.workload == "audit" else
"sequential INSERT statements over one local asyncpg connection into a temporary "
"table; table is dropped on commit; not target 6-core/32-GB hardware"
),
}
@ -90,6 +175,16 @@ if __name__ == "__main__":
default=os.environ.get("DATABASE_URL", "postgresql://lct:lct@127.0.0.1:5432/lct"),
)
parser.add_argument("--operations", type=int, default=1000)
parser.add_argument("--concurrency", type=int, default=1,
help="number of concurrent PostgreSQL clients (audit workload only)")
parser.add_argument(
"--workload", choices=("temporary", "audit"), default="temporary",
help="temporary benchmark table (default) or real indexed audit_log writes",
)
parser.add_argument(
"--allow-audit-writes", action="store_true",
help="required opt-in: workload=audit inserts then removes rows in audit_log",
)
parser.add_argument("--output", help="сохранить JSON-отчёт")
try:
raise SystemExit(asyncio.run(run(parser.parse_args())))

49
scripts/sip_recording_cleanup.py Normal file
View file

@ -0,0 +1,49 @@
"""Narrow cleanup helper for disposable SIP/WebRTC acceptance runs."""
import subprocess
from pathlib import Path
from typing import Callable
def remove_smoke_recordings(
root: Path,
names: set[str],
*,
runner: Callable[..., subprocess.CompletedProcess] | None = None,
) -> bool:
"""Remove exact WAV basenames from this Compose project's SIP volume."""
safe_names = sorted(
name for name in names
if Path(name).name == name and name.endswith(".wav")
)
if not safe_names:
return True
run = runner or subprocess.run
result = run(
[
"docker", "compose", "-f", "docker-compose.yml", "-f", "docker-compose.sip.yml",
"exec", "-T", "sip", "rm", "-f", "--",
*(f"/recordings/{name}" for name in safe_names),
],
cwd=root,
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
return False
verification = run(
[
"docker", "compose", "-f", "docker-compose.yml", "-f", "docker-compose.sip.yml",
"exec", "-T", "sip", "find", "/recordings", "-maxdepth", "1",
"-type", "f", "-name", "*.wav",
],
cwd=root,
check=False,
capture_output=True,
text=True,
)
if verification.returncode != 0:
return False
remaining = {Path(line).name for line in verification.stdout.splitlines()}
return not remaining.intersection(safe_names)

209
scripts/smoke_dds_history.py Normal file
View file

@ -0,0 +1,209 @@
"""Browser smoke for the DDS archive and instructor lesson/scenario forms."""
from __future__ import annotations
import asyncio
import json
import os
from pathlib import Path
from playwright.async_api import async_playwright
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = Path(os.environ.get("SMOKE_EVIDENCE_DIR", ROOT / ".local" / "evidence"))
BASE_URL = os.environ.get("SMOKE_BASE_URL", "http://127.0.0.1:5173").rstrip("/")
CARDS = [
{
"card_id": "36814001", "scenario_id": "t01-1", "score_auto": 75,
"score_final": 75, "ended_at": "2026-09-24T15:00:00+03:00",
"received_at": "2026-09-24T14:55:00+03:00", "title": "Медицинская помощь",
"address": "Вороновское, 3", "description": "Учебное обращение.",
"incident_type": "medical", "victims_count": 1,
"managed_service": "01", "recipient_services": ["01"],
},
{
"card_id": "36814002", "scenario_id": "t01-2", "score_auto": 90,
"score_final": 90, "ended_at": "2026-09-25T15:00:00+03:00",
"received_at": "2026-09-25T14:50:00+03:00", "title": "Пожар в квартире",
"address": "улица Учебная, 12", "description": "УМП н. — Тест 1; наблюдается дым со стороны подъезда.",
"incident_type": "fire", "victims_count": 2,
"managed_service": "01", "recipient_services": ["01", "02"],
},
]
async def main() -> None:
EVIDENCE.mkdir(parents=True, exist_ok=True)
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
page = await browser.new_page(viewport={"width": 1920, "height": 964}, device_scale_factor=1)
errors: list[str] = []
page.on("pageerror", lambda error: errors.append(str(error)))
await page.route("**/api/sessions/dds-history**", lambda route: route.fulfill(
status=200, content_type="application/json", body=json.dumps(CARDS, ensure_ascii=False),
))
await page.goto(f"{BASE_URL}/dds", wait_until="networkidle")
login = page.get_by_role("button", name="Войти в демо одним нажатием")
if await login.is_visible():
await login.click()
await page.get_by_text("Медицинская помощь", exact=True).wait_for()
history_rows = page.locator(".dds-history-registry tbody > tr:not(.dds-history-description)")
search_toggle = page.get_by_role("button", name="Открыть поиск")
searchbox = page.get_by_role("searchbox", name="Поиск по номеру, адресу или описанию")
assert await searchbox.count() == 0, "поиск должен быть свёрнут по умолчанию"
await search_toggle.click()
await searchbox.wait_for(state="visible")
await searchbox.fill("Пожар")
assert await history_rows.count() == 1
assert "Пожар в квартире" in await history_rows.first.inner_text()
await page.get_by_role("button", name="Свернуть поиск").click()
assert await searchbox.is_visible(), "активный поисковый фильтр нельзя прятать"
await searchbox.fill("")
await page.get_by_role("button", name="Свернуть поиск").click()
assert await searchbox.count() == 0, "пустой поиск можно свернуть"
await page.set_viewport_size({"width": 1280, "height": 800})
desktop_table_metrics = await page.evaluate("""() => {
const wrapper = document.querySelector('.dds-history-registry')?.parentElement;
return { viewport: innerWidth, document: document.documentElement.scrollWidth,
table: wrapper?.querySelector('table')?.scrollWidth, wrapper: wrapper?.clientWidth,
tableScrolls: !!wrapper && wrapper.scrollWidth > wrapper.clientWidth };
}""")
assert desktop_table_metrics["document"] <= desktop_table_metrics["viewport"]
assert desktop_table_metrics["tableScrolls"], desktop_table_metrics
await page.set_viewport_size({"width": 1920, "height": 964})
notification_filter = page.get_by_role("combobox", name="Фильтр уведомлений")
await notification_filter.select_option("02")
filtered_recipient_count = await history_rows.count()
assert filtered_recipient_count == 1
assert "Пожар в квартире" in await history_rows.first.inner_text()
await notification_filter.select_option("all")
default_first_date = await history_rows.first.locator("td").nth(5).inner_text()
assert default_first_date == "25.09.2026", default_first_date
sort_button = page.get_by_role("button", name="Дата ↓")
await sort_button.click()
ascending_first_date = await history_rows.first.locator("td").nth(5).inner_text()
assert ascending_first_date == "24.09.2026", ascending_first_date
await page.screenshot(path=str(EVIDENCE / "dds-history-browser-smoke-2026-09-26.png"), full_page=True)
await page.set_viewport_size({"width": 390, "height": 844})
await page.screenshot(path=str(EVIDENCE / "dds-history-mobile-smoke-2026-09-26.png"), full_page=True)
metrics = await page.evaluate("""() => {
const wrapper = document.querySelector('.dds-history-registry')?.parentElement;
return { viewport: innerWidth, document: document.documentElement.scrollWidth,
table: wrapper?.querySelector('table')?.scrollWidth, wrapper: wrapper?.clientWidth,
tableScrolls: !!wrapper && wrapper.scrollWidth > wrapper.clientWidth };
}""")
await page.set_viewport_size({"width": 1920, "height": 964})
session_list_statuses: list[int] = []
page.on("response", lambda response: session_list_statuses.append(response.status)
if response.url.split("?", 1)[0].endswith("/api/sessions") else None)
await page.goto(f"{BASE_URL}/instructor", wait_until="networkidle")
login = page.get_by_role("button", name="Войти в демо одним нажатием")
if await login.is_visible():
await login.click()
await page.get_by_text("История занятий пока пуста", exact=True).wait_for()
assert session_list_statuses and session_list_statuses[-1] == 200, session_list_statuses
scenario_filters = page.locator("details.scenario-filters-details")
await scenario_filters.wait_for()
assert not await scenario_filters.evaluate("element => element.open"), "редкие фильтры должны быть свёрнуты по умолчанию"
await page.get_by_role("searchbox", name="Поиск сценария").wait_for()
await scenario_filters.locator("summary").click()
await page.get_by_role("combobox", name="Билет заказчика").wait_for()
level_filter = page.get_by_role("combobox", name="Уровень сложности")
await level_filter.select_option("L1")
scenario_options = page.locator("select[aria-label='Сценарий занятия'] option")
await page.wait_for_function("""() => {
const select = document.querySelector("select[aria-label='Сценарий занятия']");
return select && select.options.length > 0 &&
Array.from(select.options).every(option => option.textContent.includes('L1'));
}""")
l1_option_count = await scenario_options.count()
assert l1_option_count > 0
await level_filter.select_option("")
await scenario_filters.locator("summary").click()
await page.get_by_text("Настроить нормативы и критерии оценки").click()
criterion = page.get_by_role("spinbutton", name="Лимит решения ДДС, секунд")
await criterion.wait_for()
await page.screenshot(path=str(EVIDENCE / "instructor-create-browser-smoke-2026-09-26.png"))
await criterion.fill("")
await criterion.press_sequentially("120")
entered = await criterion.input_value()
await page.get_by_role("heading", name="Рабочее место преподавателя").click()
committed = await criterion.input_value()
assert entered == "120" and committed == "120", {"typed": entered, "committed": committed}
await page.get_by_text("Настроить веса оценки (необязательно)").click()
weight = page.get_by_role("spinbutton", name="Вес метрики «Время отработки карточки»")
await weight.fill("")
await weight.press_sequentially("2.5")
weight_entered = await weight.input_value()
await page.get_by_role("heading", name="Рабочее место преподавателя").click()
weight_committed = await weight.input_value()
assert weight_entered == "2.5" and weight_committed == "2.5", {
"weightTyped": weight_entered, "weightCommitted": weight_committed,
}
draft = {
"id": "draft-smoke", "status": "draft", "generation": "template_copy",
"body": {
"id": "draft-smoke", "title": "Демо-сценарий", "type": "fire", "level": "L1",
"first_line": "Помогите, в подъезде дым!",
"facts": [{"id": "address", "value": "улица Учебная, дом 12"}],
"checklist": [{"id": "address", "question": "Где именно дым?", "fact": "улица Учебная, дом 12"}],
},
}
await page.route("**/api/scenarios/drafts/from-template", lambda route: route.fulfill(
status=200, content_type="application/json", body=json.dumps(draft, ensure_ascii=False),
))
saved_patches: list[dict] = []
async def save_draft(route):
patch = route.request.post_data_json
saved_patches.append(patch)
updated = {**draft, "body": {**draft["body"], **patch}}
await route.fulfill(status=200, content_type="application/json", body=json.dumps(updated, ensure_ascii=False))
await page.route("**/api/scenarios/drafts/draft-smoke", save_draft)
await page.route("**/api/scenarios/drafts/draft-smoke/validate", lambda route: route.fulfill(
status=200, content_type="application/json", body=json.dumps({
"valid": True, "errors": [], "ground_truth": {
"incident_type": "fire", "dds": "01", "incident_code": "01.01",
"notify": ["01"], "required_facts": ["address"],
"address": "улица Учебная, дом 12", "victims": 0,
},
}, ensure_ascii=False),
))
await page.get_by_role("button", name="Создать черновик").click()
first_line = page.get_by_role("textbox", name="Первая реплика звонящего")
await first_line.wait_for()
await first_line.fill("Помогите, в подъезде сильный дым!")
await page.get_by_role("button", name="Сохранить", exact=True).click()
await page.get_by_text("Черновик сохранён.", exact=True).wait_for()
assert saved_patches == [{"first_line": "Помогите, в подъезде сильный дым!"}], saved_patches
await page.get_by_role("button", name="Проверить", exact=True).click()
await page.get_by_text("Проверка пройдена", exact=True).wait_for()
visible_text = await page.locator("body").inner_text()
assert "JSON" not in visible_text and '{"address"' not in visible_text
await page.set_viewport_size({"width": 390, "height": 844})
instructor_mobile = await page.evaluate("""() => ({
viewport: innerWidth, document: document.documentElement.scrollWidth,
body: document.body.scrollWidth,
})""")
await page.screenshot(path=str(EVIDENCE / "instructor-create-mobile-smoke-2026-09-26.png"), full_page=True)
assert instructor_mobile["document"] <= instructor_mobile["viewport"], instructor_mobile
report = {"metrics": metrics, "desktopTableMetrics": desktop_table_metrics,
"demoSessionListStatus": session_list_statuses[-1], "criterionInput": committed,
"weightInput": weight_committed, "scenarioEditor": "no JSON shown",
"l1ScenarioOptions": l1_option_count,
"humanEditPatch": saved_patches, "instructorMobile": instructor_mobile,
"notificationFilteredRows": filtered_recipient_count,
"dateSort": {"descendingFirst": default_first_date,
"ascendingFirst": ascending_first_date},
"pageErrors": errors}
(EVIDENCE / "dds-history-controls-browser-smoke-2026-09-26.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8",
)
print(json.dumps(report, ensure_ascii=False))
await browser.close()
if metrics["document"] > metrics["viewport"] or not metrics["tableScrolls"] or errors:
raise SystemExit("DDS mobile smoke failed")
if __name__ == "__main__":
asyncio.run(main())

124
scripts/smoke_mobile.py
View file

@ -14,7 +14,7 @@ from load_browser import local_url
def layout(page) -> dict:
return page.evaluate(
"""() => {
r"""() => {
const root = document.documentElement;
const visible = [...document.querySelectorAll('body *')].filter((element) => {
const style = getComputedStyle(element);
@ -26,6 +26,12 @@ def layout(page) -> dict:
return {
viewport: innerWidth,
document_width: root.scrollWidth,
lesson_start_position: (() => {
const bar = document.querySelector('.instructor-layout .lesson-start-bar');
return bar ? getComputedStyle(bar).position : null;
})(),
visible_json: /\bjson\b/i.test(document.body.innerText)
|| /\{\s*["'][\w-]+["']\s*:/.test(document.body.innerText),
overflowing: visible.filter((element) => {
const rect = element.getBoundingClientRect();
return rect.right > innerWidth + 2 || rect.left < -2;
@ -41,7 +47,9 @@ def layout(page) -> dict:
)].filter((element) => {
const style = getComputedStyle(element);
const rect = element.getBoundingClientRect();
const closedDetails = element.closest('details:not([open])');
return style.display !== 'none' && style.visibility !== 'hidden'
&& !closedDetails
&& !element.disabled && (rect.height < 40 || rect.width < 40);
}).slice(0, 20).map((element) => ({
tag: element.tagName.toLowerCase(),
@ -73,19 +81,23 @@ def stop_exercise_from_instructor(page, base_url: str, exercise_href: str) -> No
def choose_trainee(page) -> None:
# Match the label cell, not any row whose body happens to mention a
# trainee (the scenario/category tables include that word too).
# Match the step title even when mobile layout renders the step number
# inside the same cell. Do not use exact text on the nested th: its actual
# accessible text is now "4 Курсант".
exercise_table = page.locator(".instructor-layout > section").first.locator(
"table.grid"
).first
trainee_row = exercise_table.locator("th").get_by_text(
"Курсант", exact=True
).locator("xpath=..")
trainee_row = exercise_table.locator(".lesson-step-title").filter(
has_text="Курсант"
).locator("xpath=../..")
trainee_select = trainee_row.locator("select")
if trainee_select.count():
if trainee_select.locator("option").count() < 2:
raise RuntimeError("Список курсантов не содержит доступного выбора")
trainee_select.select_option(index=1)
demo_option = trainee_select.locator('option[value="__demo__"]')
trainee_select.select_option(
value="__demo__" if demo_option.count() else trainee_select.locator("option").nth(1).get_attribute("value")
)
else:
trainee_row.locator('input[type="text"]').fill("Мобильный smoke")
@ -117,6 +129,11 @@ def exercise_dds_flow(page, scenario_title: str) -> dict:
if not any("Принята" in label for label in labels):
raise RuntimeError(f"статус «Принята» недоступен для {service_title}: {labels}")
next_status.select_option(value="accepted")
service_name = next_status.get_attribute("aria-label").removeprefix("Следующий статус ")
page.get_by_role("textbox", name=f"Основание статуса {service_name}").fill("Учебный доклад бригады")
page.get_by_role("textbox", name=f"Полученные сведения {service_name}").fill(
"Служба приняла карточку и направила подразделение к месту происшествия."
)
page.locator('button[aria-label="Сохранить статус"]').click()
page.wait_for_function("""() => [...document.querySelectorAll(
'.dds-service-dock button')].some((button) => button.innerText.includes('Принята'))""")
@ -164,9 +181,10 @@ def smoke_dds_queue(page, base_url: str, output: Path, screenshot_suffix: str) -
queue = page.locator("details.scenario-queue")
queue.locator("summary").click()
queue.locator('input[type="checkbox"]').first.check()
page.get_by_text("Настроить нормативы и критерии оценки").click()
page.get_by_label("Интервал поступления карточек ДДС, секунд").fill("6")
page.get_by_label("Максимум ожидающих карточек ДДС").fill("1")
dds_href = start_exercise(page, "Диспетчер ДДС · готовая карточка", "АРМ ДДС")
dds_href = start_exercise(page, "Диспетчер ДДС", "АРМ ДДС")
page.goto(dds_href, wait_until="networkidle")
page.locator(".dds-workspace").wait_for()
first_row = page.locator('.dds-registry tbody tr[aria-current="true"]')
@ -185,6 +203,10 @@ def smoke_dds_queue(page, base_url: str, output: Path, screenshot_suffix: str) -
if not any("Принята" in option for option in next_status.locator("option").all_text_contents()):
raise RuntimeError(f"для первой карточки недоступен статус «Принята»: {service_title}")
next_status.select_option(value="accepted")
page.get_by_role("textbox", name=f"Основание статуса {service_name}").fill("Учебный доклад бригады")
page.get_by_role("textbox", name=f"Полученные сведения {service_name}").fill(
"Служба приняла карточку и направила подразделение к месту происшествия."
)
page.locator('button[aria-label="Сохранить статус"]').click()
page.wait_for_function(
"""(service) => [...document.querySelectorAll('.dds-service-dock button')]
@ -193,7 +215,7 @@ def smoke_dds_queue(page, base_url: str, output: Path, screenshot_suffix: str) -
arg=service_title,
timeout=5_000,
)
page.locator(".dds-case-top button").click()
page.locator(".dds-card-back").click()
page.wait_for_function(
"document.querySelectorAll('.dds-registry tbody tr').length === 2",
timeout=20_000,
@ -204,18 +226,18 @@ def smoke_dds_queue(page, base_url: str, output: Path, screenshot_suffix: str) -
second_row = rows.nth(1)
second_number = second_row.locator('td[data-label="Номер"] button').inner_text()
second_row.get_by_role("button", name="Открыть карточку").click()
page.locator(".dds-case-top").get_by_text(second_number, exact=False).wait_for()
page.locator(".dds-case-top button").click()
page.locator(".dds-call-incident").get_by_text(second_number, exact=False).wait_for()
page.locator(".dds-card-back").click()
page.locator(".dds-registry tbody tr").filter(has_text=first_number).get_by_role(
"button", name="Открыть карточку"
).click()
page.locator(".dds-case-top").get_by_text(first_number, exact=False).wait_for()
page.locator(".dds-call-incident").get_by_text(first_number, exact=False).wait_for()
restored_status = page.locator(".dds-service-dock button").filter(
has_text=service_title
).first.inner_text()
if "Принята" not in restored_status:
raise RuntimeError(f"статус первой карточки потерян после поступления/переключения: {restored_status}")
page.locator(".dds-case-top button").click()
page.locator(".dds-card-back").click()
screen = layout(page)
page.screenshot(path=output / f"dds-queue-{screenshot_suffix}.png", full_page=True)
stop_exercise_from_instructor(page, base_url, dds_href)
@ -234,6 +256,8 @@ def main() -> int:
parser.add_argument("--report", type=Path)
parser.add_argument("--login")
parser.add_argument("--password")
parser.add_argument("--results-only", action="store_true",
help="stop after verifying the completed lesson report UI")
parser.add_argument("--width", type=int, default=390)
parser.add_argument("--height", type=int, default=844)
parser.add_argument("--device-scale-factor", type=float, default=2)
@ -264,6 +288,8 @@ def main() -> int:
locale="ru-RU",
)
page = context.new_page()
browser_errors: list[str] = []
page.on("pageerror", lambda error: browser_errors.append(str(error)))
page.goto(f"{args.base_url}/instructor", wait_until="networkidle")
demo_login = page.get_by_role("button", name="Войти в демо одним нажатием")
if demo_login.is_visible():
@ -277,6 +303,24 @@ def main() -> int:
page.locator("#ai-scenario-editor").wait_for()
page.screenshot(path=args.output / f"instructor-{screenshot_suffix}.png", full_page=True)
report["screens"]["instructor"] = layout(page)
step_titles = [
" ".join(value.split())
for value in page.locator(".lesson-step-title").all_inner_texts()
]
if step_titles[:4] != [
"1 Сценарий", "2 Упражнение", "3 Режим", "4 Курсант",
]:
raise RuntimeError(f"Шаги создания занятия отображаются не по порядку: {step_titles[:4]}")
page.get_by_label("Поиск сценария").fill("smoke-no-matching-scenario")
page.locator(".scenario-empty-state").wait_for()
page.get_by_role("button", name="Сбросить фильтры").last.click()
page.locator(".scenario-empty-state").wait_for(state="hidden")
page.locator(".selected-scenario-summary").wait_for()
report["workflow"]["lesson_setup"] = {
"ordered_steps": step_titles[:4],
"empty_search_explains_next_action": True,
"filter_reset_restores_scenario": True,
}
trainee_href = start_exercise(
page, "Оператор 112 · текстовая вводная", "АРМ курсанта"
@ -286,10 +330,51 @@ def main() -> int:
page.screenshot(path=args.output / f"trainee-{screenshot_suffix}.png", full_page=True)
report["screens"]["trainee"] = layout(page)
stop_exercise_from_instructor(page, args.base_url, trainee_href)
results_heading = page.get_by_role("heading", name="Результаты занятия")
results_heading.wait_for()
if not page.locator(".debrief-summary").is_visible():
raise RuntimeError("после завершения занятия не показана сводка результатов")
technical_details = page.locator(".debrief-tech-details")
technical_open = technical_details.evaluate("element => element.open")
if technical_open:
raise RuntimeError("технические сведения должны быть скрыты до раскрытия")
results_text = page.locator(".debrief").last.inner_text()
if "JSON" in results_text:
raise RuntimeError("в итогах занятия отображается JSON")
page.screenshot(path=args.output / f"instructor-results-{screenshot_suffix}.png", full_page=True)
report["screens"]["instructor_results"] = layout(page)
report["workflow"]["results_summary"] = {
"visible_after_finish": True,
"technical_details_collapsed": not technical_open,
"json_visible": False,
}
if args.results_only:
browser.close()
results_screen = report["screens"]["instructor_results"]
report["passed"] = (
results_screen["document_width"] <= results_screen["viewport"] + 2
and results_screen["small_controls"] == []
and not results_screen["visible_json"]
and results_screen["lesson_start_position"] in (None, "static")
and not browser_errors
)
report["browser_errors"] = browser_errors
report["problems"] = {
"instructor_results_horizontal_overflow": not report["passed"],
"instructor_results_small_controls": results_screen["small_controls"],
"instructor_results_visible_json": results_screen["visible_json"],
"browser_errors": browser_errors,
}
rendered = json.dumps(report, ensure_ascii=False, indent=2)
print(rendered)
if args.report:
args.report.parent.mkdir(parents=True, exist_ok=True)
args.report.write_text(rendered + "\n", encoding="utf-8")
return 0 if report["passed"] else 1
open_instructor(page, args.base_url)
dds_href = start_exercise(
page, "Диспетчер ДДС · готовая карточка", "АРМ ДДС"
page, "Диспетчер ДДС", "АРМ ДДС"
)
page.goto(dds_href, wait_until="networkidle")
page.locator(".dds-workspace").wait_for()
@ -304,7 +389,7 @@ def main() -> int:
page.locator(".dds-case-layout").wait_for()
page.screenshot(path=args.output / f"dds-card-{screenshot_suffix}.png", full_page=True)
report["screens"]["dds_card"] = layout(page)
report["workflow"] = exercise_dds_flow(page, scenario_title)
report["workflow"].update(exercise_dds_flow(page, scenario_title))
page.screenshot(path=args.output / f"dds-workflow-{screenshot_suffix}.png", full_page=True)
report["screens"]["dds_workflow"] = layout(page)
page.locator(".dds-service-dock button").last.click()
@ -319,16 +404,19 @@ def main() -> int:
problems = {
name: {
"horizontal_overflow": data["document_width"] > data["viewport"] + 2,
"visible_json": data["visible_json"],
"small_controls": len(data["small_controls"]),
"start_button_overlay": data["lesson_start_position"] not in (None, "static"),
"touch_target_check_applies": args.width <= 600,
}
for name, data in report["screens"].items()
}
report["problems"] = problems
report["passed"] = all(
not item["horizontal_overflow"] and (
report["browser_errors"] = browser_errors
report["passed"] = not browser_errors and all(
not item["horizontal_overflow"] and not item["visible_json"] and (
not item["touch_target_check_applies"] or item["small_controls"] == 0
)
) and not item["start_button_overlay"]
for item in problems.values()
)
rendered = json.dumps(report, ensure_ascii=False, indent=2)

268
scripts/smoke_student_dds_browser.py Normal file
View file

@ -0,0 +1,268 @@
#!/usr/bin/env python3
"""Live-browser acceptance of trainee KIO -> instructor review -> DDS status.
Only use with a disposable loopback stack/database. The script creates unique
accounts and one group, then removes every row it created in a finally block.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import re
import secrets
import sys
import time
from pathlib import Path
from urllib.parse import urlsplit
from uuid import UUID
from playwright.async_api import async_playwright
from sqlalchemy import delete, or_, select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
def require_loopback(value: str, name: str) -> None:
if urlsplit(value).hostname not in {"127.0.0.1", "localhost", "::1"}:
raise ValueError(f"{name} must use loopback")
async def login(page, base: str, user: str, password: str, target: str) -> None:
await page.goto(f"{base}{target}", wait_until="domcontentloaded")
login_heading = page.get_by_role("heading", name="Учебный симулятор системы-112")
if await login_heading.is_visible():
await page.get_by_label("Логин").fill(user)
await page.get_by_label("Пароль").fill(password)
await page.get_by_role("button", name="Войти", exact=True).click()
await page.get_by_role("heading", name={"/profile": "Профиль курсанта", "/instructor": "Рабочее место преподавателя"}[target]).wait_for()
async def main(args: argparse.Namespace) -> int:
require_loopback(args.frontend_url, "frontend-url")
require_loopback(args.database_url, "database-url")
from app.api.auth import hash_password
from app.db.models import AuditLog, Group, Scenario as ScenarioRow, ScenarioSubmission, Session, Trainee, User
from app.domain import ekp
from app.scenarios import store
scenarios = store.load_from_disk(args.scenarios)
source = next((item for item in scenarios if item.id == args.source_scenario), None)
if source is None or source.ground_truth.incident_code is None or source.ground_truth.dds is None:
raise ValueError(f"source scenario must be a classified DDS case: {args.source_scenario}")
incident_group = ekp.incident(source.ground_truth.incident_code).group
service = source.ground_truth.dds.value
suffix = secrets.token_hex(5)
teacher_login = f"browser-i-{suffix}"
trainee_login = f"browser-t-{suffix}"
teacher_password = secrets.token_urlsafe(24)
trainee_password = secrets.token_urlsafe(24)
teacher_name = f"Browser instructor {suffix}"
trainee_name = f"Browser trainee {suffix}"
group = Group(name=f"browser-smoke-{suffix}", owner_login=teacher_login)
trainee = Trainee(name=trainee_name)
session_ids: list[UUID] = []
submission_id: UUID | None = None
published_scenario_id: str | None = None
session_id: UUID | None = None
result: dict = {"checks": {}, "source_scenario": source.id}
engine = create_async_engine(args.database_url, pool_pre_ping=True)
factory = async_sessionmaker(engine, expire_on_commit=False)
try:
async with factory() as db:
db.add(group)
await db.flush()
trainee.group_id = group.id
db.add(trainee)
await db.flush()
db.add_all([
User(login=teacher_login, full_name=teacher_name,
password_hash=hash_password(teacher_password), role="instructor", blocked=False),
User(login=trainee_login, full_name=trainee_name,
password_hash=hash_password(trainee_password), role="trainee",
trainee_id=trainee.id, service=service, blocked=False),
])
await db.commit()
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
student_page = await browser.new_page(ignore_https_errors=True)
await login(student_page, args.frontend_url, trainee_login, trainee_password, "/profile")
title = f"КИО из браузера {suffix}"
address = f"Москва, учебная улица, дом {suffix[:2]}"
description = "В учебном помещении виден дым, требуется проверка и выезд службы."
await student_page.get_by_label("Название").fill(title)
await student_page.locator('input[aria-label="Адрес"]').fill(address)
await student_page.locator('textarea[aria-label="Описание"]').fill(description)
await student_page.get_by_label("Категория происшествия").select_option(value=str(incident_group))
await student_page.get_by_role("group", name="Что случилось?").get_by_role(
"button", name="Пожар", exact=True
).click()
for index, sign in enumerate(source.signs, start=1):
picker = student_page.get_by_label(f"Признак происшествия {index}")
await picker.wait_for(state="visible")
await picker.select_option(label=sign)
await student_page.get_by_role("button", name="Сохранить черновик преподавателю").click()
await student_page.get_by_text("Черновик КИО отправлен и ожидает проверки.", exact=True).wait_for()
result["checks"]["student_submitted_kio_from_profile"] = True
async with factory() as db:
deadline = time.monotonic() + 10
submission = None
while time.monotonic() < deadline:
submission = await db.scalar(select(ScenarioSubmission).where(ScenarioSubmission.title == title))
if submission is not None:
break
await asyncio.sleep(0.1)
if submission is None:
raise RuntimeError("browser-submitted KIO was not persisted in PostgreSQL")
submission_id = submission.id
instructor_page = await browser.new_page(ignore_https_errors=True)
await login(instructor_page, args.frontend_url, teacher_login, teacher_password, "/instructor")
proposal = instructor_page.locator(".student-scenario-proposal").filter(has_text=title)
await proposal.wait_for(state="visible")
await proposal.get_by_role("button", name="Проверил(а), утвердить в банк ДДС").click()
await proposal.wait_for(state="hidden")
result["checks"]["instructor_approved_in_browser"] = True
await instructor_page.get_by_label("Источник карточек").select_option(label="Сформированные курсантами")
scenario_select = instructor_page.get_by_label("Сценарий занятия")
await scenario_select.wait_for()
deadline = time.monotonic() + 10
selected_value = None
while time.monotonic() < deadline:
options = await scenario_select.locator("option").all()
for option in options:
if title in (await option.text_content() or ""):
selected_value = await option.get_attribute("value")
break
if selected_value:
break
await asyncio.sleep(0.1)
if not selected_value:
raise RuntimeError("approved trainee KIO did not enter instructor's DDS scenario bank")
published_scenario_id = selected_value
await scenario_select.select_option(value=selected_value)
trainee_select = instructor_page.get_by_label("Учётная запись курсанта")
trainee_options = await trainee_select.locator("option").all()
trainee_value = None
for option in trainee_options:
if trainee_name in (await option.text_content() or ""):
trainee_value = await option.get_attribute("value")
break
if not trainee_value:
raise RuntimeError("temporary trainee account is missing from instructor's trainee list")
await trainee_select.select_option(value=trainee_value)
await instructor_page.get_by_role("button", name="Начать занятие").click()
await instructor_page.wait_for_function("new URL(location.href).searchParams.has('session')", timeout=15000)
session_id = UUID(urlsplit(instructor_page.url).query.split("session=", 1)[1].split("&", 1)[0])
session_ids.append(session_id)
await instructor_page.get_by_role("link", name=re.compile("открыть ДДС")).wait_for()
result["checks"]["instructor_started_dds_lesson"] = True
await student_page.goto(f"{args.frontend_url}/dds?session={session_id}&role=dds_{service}", wait_until="domcontentloaded")
await student_page.locator(".dds-connection-live").filter(has_text="на связи").wait_for(timeout=15000)
queue_row = student_page.locator("tr").filter(has_text=title)
await queue_row.wait_for(state="visible", timeout=15000)
await queue_row.get_by_role("button", name="Открыть карточку").click()
await student_page.locator(".dds-case-address").get_by_text(address, exact=True).wait_for()
result["checks"]["approved_kio_received_in_live_dds"] = True
service_buttons = student_page.locator(".dds-service-recipient-list .dds-service-recipient")
await service_buttons.first.wait_for(state="visible")
result["recipient_buttons"] = await service_buttons.all_text_contents()
managed_button = None
for index in range(await service_buttons.count()):
candidate = service_buttons.nth(index)
if await candidate.is_enabled():
managed_button = candidate
break
if managed_button is None:
raise RuntimeError(f"no manageable DDS service button: {result['recipient_buttons']}")
managed_button_text = await managed_button.locator("b").inner_text()
await managed_button.click()
classification = await student_page.locator(".dds-case-classification").inner_text()
result["actual_managed_service_button"] = " ".join(managed_button_text.split())
result["service_assignment_matches_card"] = result["actual_managed_service_button"] in classification
if not result["service_assignment_matches_card"]:
raise RuntimeError("enabled managed-service button differs from the service shown on the card")
await student_page.locator(".dds-case-drawer-status").wait_for(state="visible", timeout=5000)
status_controls = student_page.locator('select[aria-label^="Следующий статус "]')
if await status_controls.count() == 0:
raise RuntimeError("DDS status pane opened without a status control: " +
(await student_page.locator("body").inner_text())[-1600:])
status_select = status_controls.first
await status_select.select_option(label="Принята")
await student_page.get_by_label("Основание статуса").fill("Карточка получена и принята ДДС.")
await student_page.get_by_label("Полученные сведения").fill("Направление реагирования подтверждено.")
await student_page.get_by_role("button", name="Сохранить статус").click()
await student_page.locator(".dds-status-history").get_by_text("Принята", exact=False).first.wait_for()
result["checks"]["dds_primary_status_saved"] = True
crew_select = student_page.get_by_label(f"Наряд {service}")
crew_options = await crew_select.locator("option").all()
crew_value = None
for option in crew_options:
candidate = await option.get_attribute("value")
if candidate:
crew_value = candidate
break
if crew_value:
await crew_select.select_option(value=crew_value)
await status_select.select_option(label="Начало реагирования")
await student_page.get_by_label("Основание статуса").fill("Получен доклад о начале реагирования.")
await student_page.get_by_label("Полученные сведения").fill("Бригада выехала к месту происшествия.")
await student_page.get_by_role("button", name="Сохранить статус").click()
history = student_page.locator(".dds-status-history")
await history.get_by_text("Начало реагирования", exact=False).first.wait_for()
result["checks"]["dds_followup_status_saved"] = True
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
await student_page.screenshot(path=args.output, full_page=True)
result["session_id"] = str(session_id)
result["scenario_id"] = published_scenario_id
result["managed_service"] = service
result["all_checks_passed"] = all(result["checks"].values())
await browser.close()
rendered = json.dumps(result, ensure_ascii=False, indent=2)
print(rendered)
if args.json_output:
Path(args.json_output).parent.mkdir(parents=True, exist_ok=True)
Path(args.json_output).write_text(rendered + "\n", encoding="utf-8")
return 0 if result["all_checks_passed"] else 1
finally:
async with factory() as db:
await db.execute(delete(Session).where(or_(
Session.trainee_id == trainee.id,
Session.id.in_(session_ids) if session_ids else False,
)))
if submission_id is not None:
await db.execute(delete(ScenarioSubmission).where(ScenarioSubmission.id == submission_id))
if published_scenario_id:
await db.execute(delete(ScenarioRow).where(ScenarioRow.id == published_scenario_id))
await db.execute(delete(AuditLog).where(AuditLog.actor.in_([teacher_login, trainee_login])))
await db.execute(delete(User).where(User.login.in_([teacher_login, trainee_login])))
await db.execute(delete(Trainee).where(Trainee.name == trainee_name))
await db.execute(delete(Group).where(Group.name == group.name))
await db.commit()
await engine.dispose()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--frontend-url", required=True)
parser.add_argument("--database-url", required=True)
parser.add_argument("--scenarios", type=Path, default=ROOT / "scenarios")
parser.add_argument("--source-scenario", default="t01-1-fire-container")
parser.add_argument("--output", help="optional DDS screenshot path")
parser.add_argument("--json-output", help="optional machine-readable evidence path")
raise SystemExit(asyncio.run(main(parser.parse_args())))

33
scripts/smoke_webrtc_browser.py
View file

@ -17,6 +17,7 @@ from sqlalchemy import delete
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from load_browser import local_url, login_cookie
from sip_recording_cleanup import remove_smoke_recordings
from smoke_sip import compose_password
ROOT = Path(__file__).resolve().parents[1]
@ -89,12 +90,14 @@ async def run(args: argparse.Namespace) -> int:
login = f"webrtc-smoke-{secrets.token_hex(5)}"
app_password = secrets.token_urlsafe(24)
await create_account(args.database_url, login, app_password)
recordings_before: set[str] | None = None
try:
cookie = login_cookie(backend, login, app_password)
cookie_name, cookie_value = cookie.split("=", 1)
sip_6101 = compose_password("6101")
sip_6102 = compose_password("6102")
before = recordings()
recordings_before = set(before)
errors: list[str] = []
started = time.perf_counter()
timeout_ms = args.timeout * 1000
@ -129,7 +132,19 @@ async def run(args: argparse.Namespace) -> int:
)
registered_ms = (time.perf_counter() - started) * 1000
await pages[0].get_by_role("button", name="Позвонить").click()
await pages[1].get_by_role("button", name="Ответить · 6101").wait_for(timeout=timeout_ms)
try:
await pages[1].get_by_role("button", name="Ответить · 6101").wait_for(timeout=timeout_ms)
except Exception as exc:
diagnostics = []
for page in pages:
diagnostics.append({
"url": page.url,
"body": (await page.locator("body").inner_text())[-3000:],
})
raise RuntimeError(
"incoming SIP call was not displayed; browser states: "
+ json.dumps(diagnostics, ensure_ascii=False)
) from exc
await pages[1].get_by_role("button", name="Ответить · 6101").click()
await asyncio.gather(
pages[0].get_by_text("разговор идёт", exact=False).wait_for(timeout=timeout_ms),
@ -185,7 +200,21 @@ async def run(args: argparse.Namespace) -> int:
args.output.write_text(rendered + "\n", encoding="utf-8")
return 0 if passed else 1
finally:
await cleanup_account(args.database_url, login)
try:
if recordings_before is not None:
try:
current_recordings = recordings()
except (OSError, subprocess.CalledProcessError):
current_recordings = {}
print("WARNING: не удалось проверить очистку временной WAV-записи WebRTC smoke", file=sys.stderr)
created = set(current_recordings) - recordings_before
# Only this exercise's direction; preserve older files and any
# unrelated recordings in the stack's durable volume.
created = {name for name in created if "-6101-6102-" in name}
if created and not remove_smoke_recordings(ROOT, created):
print("WARNING: не удалось удалить временную WAV-запись WebRTC smoke", file=sys.stderr)
finally:
await cleanup_account(args.database_url, login)
if __name__ == "__main__":

645
scripts/test_cluster_failover.py Normal file
View file

@ -0,0 +1,645 @@
#!/usr/bin/env python3
"""Live smoke: verify a complete DDS exercise across fenced cluster takeover.
Only use with a disposable local Compose cluster and its own PostgreSQL database.
The script creates temporary users/session, stops and restarts one named backend,
and removes the temporary rows on completion.
"""
from __future__ import annotations
import argparse
import asyncio
from contextlib import AsyncExitStack
import json
import os
import secrets
import ssl
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
from uuid import UUID, uuid4
import websockets
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "backend"))
def compose(args: argparse.Namespace, *command: str, check: bool = True) -> str:
env = os.environ.copy()
env.update({
"POSTGRES_PORT": str(args.postgres_port),
"BACKEND_PORT": str(args.backend_port),
"BACKEND_B_PORT": str(args.backend_b_port),
"FRONTEND_PORT": str(args.frontend_port),
"TLS_PORT": str(args.tls_port),
"POSTGRES_PASSWORD": args.postgres_password,
"SESSION_SECRET": args.session_secret,
"TLS_CERT_DIR": args.tls_cert_dir,
})
files = [
"docker-compose.yml", "docker-compose.tls.yml",
"docker-compose.load-test.yml", "docker-compose.cluster.yml",
]
if args.failure_mode == "partition":
files.append("docker-compose.partition-test.yml")
result = subprocess.run(
["docker", "compose", "-p", args.project,
*[part for file in files for part in ("-f", str(ROOT / file))], *command],
cwd=ROOT, env=env, check=False, capture_output=True, text=True,
)
if check and result.returncode:
raise subprocess.CalledProcessError(
result.returncode, result.args, output=result.stdout, stderr=result.stderr
)
return result.stdout
def login(base: str, login_name: str, password: str) -> str:
payload = json.dumps({"login": login_name, "password": password}).encode()
request = urllib.request.Request(
f"{base}/api/auth/login", data=payload, method="POST",
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(request, timeout=10) as response:
cookie = response.headers.get("Set-Cookie", "").split(";", 1)[0]
if not cookie.startswith("lct_session="):
raise RuntimeError("login did not issue lct_session cookie")
return cookie
async def send_control(ws_base: str, session_id: UUID, cookie: str, payload: dict) -> None:
async with websockets.connect(
f"{ws_base}/ws/control/{session_id}",
additional_headers={"Cookie": cookie},
ssl=ssl._create_unverified_context() if ws_base.startswith("wss://") else None,
open_timeout=10,
ping_interval=None,
) as socket:
await socket.send(json.dumps(payload, ensure_ascii=False))
await asyncio.sleep(0.15)
async def probe_ws(ws_base: str, path: str, cookie: str) -> dict:
try:
async with websockets.connect(
f"{ws_base}{path}", additional_headers={"Cookie": cookie},
ssl=ssl._create_unverified_context() if ws_base.startswith("wss://") else None,
open_timeout=6, ping_interval=None,
) as socket:
try:
message = await asyncio.wait_for(socket.recv(), timeout=4)
event = json.loads(message) if isinstance(message, str) else "binary"
except TimeoutError:
event = "connected_no_initial_event"
return {"connected": True, "initial_event": event}
except Exception as exc: # report exact endpoint failure in the result
return {"connected": False, "error": f"{type(exc).__name__}: {exc}"}
async def station_command(
ws_base: str, session_id: UUID, cookie: str, payload: dict | None = None,
) -> dict:
"""Read the authoritative station snapshot, optionally issue one command."""
async with websockets.connect(
f"{ws_base}/ws/station/{session_id}", additional_headers={"Cookie": cookie},
ssl=ssl._create_unverified_context() if ws_base.startswith("wss://") else None,
open_timeout=10, ping_interval=None,
) as socket:
async def receive_snapshot() -> dict:
for _ in range(60):
raw = await asyncio.wait_for(socket.recv(), timeout=10)
event = json.loads(raw) if isinstance(raw, str) else {}
if event.get("type") == "error":
raise RuntimeError(f"station rejected smoke: {event.get('message')}")
if event.get("type") == "station.state":
return event["snapshot"]
raise RuntimeError("station did not send its state snapshot")
snapshot = await receive_snapshot()
if payload is None:
return snapshot
await socket.send(json.dumps(payload, ensure_ascii=False))
return await receive_snapshot()
async def replay_station_command(
ws_base: str, session_id: UUID, cookie: str, payload: dict, command_id: str,
attempts: int = 5,
) -> bool:
"""Replay one already committed command and require its durable ACK."""
last_error = None
for attempt in range(attempts):
try:
async with websockets.connect(
f"{ws_base}/ws/station/{session_id}", additional_headers={"Cookie": cookie},
ssl=ssl._create_unverified_context() if ws_base.startswith("wss://") else None,
open_timeout=10, ping_interval=None,
) as socket:
for _ in range(60):
raw = await asyncio.wait_for(socket.recv(), timeout=10)
event = json.loads(raw) if isinstance(raw, str) else {}
if event.get("type") == "error":
raise RuntimeError(f"station reconnect rejected: {event.get('message')}")
if event.get("type") == "station.state":
break
else:
raise RuntimeError("replacement station did not send a state snapshot")
replay = {**payload, "_command_id": command_id}
await socket.send(json.dumps(replay, ensure_ascii=False))
for _ in range(60):
raw = await asyncio.wait_for(socket.recv(), timeout=10)
event = json.loads(raw) if isinstance(raw, str) else {}
if event.get("type") == "error":
raise RuntimeError(f"station replay rejected: {event.get('message')}")
if event.get("type") == "command.ack":
return event.get("command_id") == command_id
raise RuntimeError("replacement owner did not acknowledge replayed command")
except (TimeoutError, OSError, websockets.exceptions.WebSocketException) as exc:
last_error = exc
if attempt + 1 < attempts:
await asyncio.sleep(1)
raise RuntimeError(f"station replay did not reconnect: {last_error}")
def ws_connect(ws_base: str, path: str, cookie: str):
return websockets.connect(
f"{ws_base}{path}", additional_headers={"Cookie": cookie},
ssl=ssl._create_unverified_context() if ws_base.startswith("wss://") else None,
open_timeout=8, ping_interval=None,
)
async def wait_for_fence(socket, timeout: float = 10) -> dict:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
message = await asyncio.wait_for(socket.recv(), timeout=deadline - time.monotonic())
except TimeoutError:
return {"closed": False, "fence_notice": False, "reason": "timeout"}
except Exception as exc:
close = getattr(exc, "rcvd", None) or getattr(exc, "sent", None)
close_code = getattr(close, "code", None) or getattr(exc, "code", None)
return {
"closed": True,
"fence_notice": close_code == 1012,
"close_code": close_code,
"reason": getattr(close, "reason", "") if close else str(exc),
}
if isinstance(message, str):
try:
event = json.loads(message)
except json.JSONDecodeError:
continue
if event.get("message") == "Занятие передано другому backend-узлу; переподключитесь.":
return {"closed": True, "fence_notice": True, "close_code": 1012,
"reason": event.get("message")}
return {"closed": False, "fence_notice": False, "reason": "timeout"}
async def main(args: argparse.Namespace) -> int:
from sqlalchemy import delete, select
from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import NullPool
from app.api.auth import hash_password
from app.db.models import AuditLog, Session, Trainee, User
database_url = (
f"postgresql+asyncpg://lct:{args.postgres_password}@127.0.0.1:"
f"{args.postgres_port}/lct"
)
if make_url(database_url).host not in {"127.0.0.1", "localhost"}:
raise ValueError("failover smoke accepts loopback PostgreSQL only")
engine = create_async_engine(database_url)
factory = async_sessionmaker(engine, expire_on_commit=False)
observer_engine = create_async_engine(database_url, poolclass=NullPool)
run_id = uuid4().hex[:12]
session_id = uuid4()
trainee = Trainee(name=f"Failover smoke {run_id}")
instructor_login = f"fo-i-{run_id}"
trainee_login = f"fo-t-{run_id}"
instructor_password = secrets.token_urlsafe(24)
trainee_password = secrets.token_urlsafe(24)
owner_service = "backend"
owner_stopped = False
fault_service = None
fault_stopped = False
result: dict = {"session_id": str(session_id), "checks": {}}
backend = f"http://127.0.0.1:{args.backend_port}"
ws_base = args.frontend_url.replace("https://", "wss://").replace("http://", "ws://")
instructor_cookie = trainee_cookie = None
try:
async with factory() as db:
db.add(trainee)
await db.flush()
db.add_all([
User(login=instructor_login, full_name="Failover smoke instructor",
password_hash=hash_password(instructor_password), role="instructor",
blocked=False),
User(login=trainee_login, full_name=trainee.name,
password_hash=hash_password(trainee_password), role="trainee",
trainee_id=trainee.id, blocked=False),
])
await db.commit()
instructor_cookie = login(backend, instructor_login, instructor_password)
trainee_cookie = login(backend, trainee_login, trainee_password)
await send_control(ws_base, session_id, instructor_cookie, {
"type": "scenario.start", "scenario_id": args.scenario,
"trainee": trainee.name, "trainee_id": str(trainee.id),
"mode": "training", "exercise": "dds",
})
async def session_row():
async with factory() as db:
return await db.scalar(select(Session).where(Session.id == session_id))
async def wait_for_command_checkpoint(command_id: str, timeout: float = 10) -> bool:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
# Use a new physical connection, avoiding a pooled observer's
# stale read transaction while the websocket writer commits.
async with observer_engine.connect() as db:
payload = (await db.execute(
select(Session.live_state).where(Session.id == session_id)
)).scalar_one_or_none()
if payload and command_id in payload.get("processed_station_commands", []):
return True
await asyncio.sleep(0.1)
return False
deadline = time.monotonic() + 12
row = None
while time.monotonic() < deadline:
row = await session_row()
if row and row.backend_node_id and row.checkpoint_at and row.live_state:
break
await asyncio.sleep(0.2)
if not row or not row.live_state:
raise RuntimeError("session did not persist an owner and recovery checkpoint")
if row.backend_node_id not in {"backend-a", "backend-b"}:
raise RuntimeError(f"unexpected session owner: {row.backend_node_id}")
owner_service = "backend" if row.backend_node_id == "backend-a" else "backend-b"
old_epoch = row.backend_fencing_epoch
result["initial_owner"] = row.backend_node_id
result["initial_epoch"] = old_epoch
# Commit real DDS work before the fault. These actions must survive the
# checkpoint handoff and remain part of the final scored report.
snapshot = await station_command(ws_base, session_id, trainee_cookie)
service = snapshot.get("managed_service") or next(iter(snapshot["services"]), None)
if not service:
raise RuntimeError("the DDS exercise has no managed service")
crew = next(
(item for item in snapshot["crew_options"] if item.startswith(service + " — ")),
None,
)
if not crew:
raise RuntimeError(f"no crew option is available for {service}")
steps_before_fault = []
for status, detail in (
("accepted", "card accepted for processing"),
("crew.select", crew),
("responding", "crew reported departure"),
):
if status == "crew.select":
command = {"type": status, "crew": crew}
else:
command = {
"type": "card.status", "service": service, "status": status,
"comment": f"Основание: доклад по карточке.\nСведения: {detail}; {service} notified.",
}
snapshot = await station_command(ws_base, session_id, trainee_cookie, command)
steps_before_fault.append(status)
result["dds_progress_before_fault"] = steps_before_fault
result["dds_service"] = service
result["dds_crew"] = crew
# Commit a non-replace station command, but close the client socket
# without reading command.ack. The live database read proves the ID
# and business state are committed before the owner fails.
lost_ack_command_id = str(uuid4())
lost_ack_command = {
"type": "card.status", "service": service, "status": "arrived",
"comment": (
"Основание: доклад по карточке.\n"
f"Сведения: прибытие до отказа; {service} notified."
),
}
async with ws_connect(
ws_base, f"/ws/station/{session_id}", trainee_cookie,
) as lost_ack_socket:
for _ in range(60):
raw = await asyncio.wait_for(lost_ack_socket.recv(), timeout=10)
event = json.loads(raw) if isinstance(raw, str) else {}
if event.get("type") == "station.state":
break
if event.get("type") == "error":
raise RuntimeError(f"station rejected lost-ACK setup: {event.get('message')}")
else:
raise RuntimeError("station did not become ready for lost-ACK command")
await lost_ack_socket.send(json.dumps({
**lost_ack_command, "_command_id": lost_ack_command_id,
}, ensure_ascii=False))
# Observe protocol output in the harness, but deliberately do not
# dispatch the ACK to the simulated browser/outbox. This is the
# lost-ack boundary: the command is committed, client state stays
# pending, and the socket is closed before the application consumes
# command.ack.
deadline = time.monotonic() + 12
ack_seen = False
while time.monotonic() < deadline:
raw = await asyncio.wait_for(
lost_ack_socket.recv(), timeout=max(0.1, deadline - time.monotonic())
)
event = json.loads(raw) if isinstance(raw, str) else {}
if event.get("type") == "error":
raise RuntimeError(
f"station command failed before lost-ACK simulation: {event.get('message')}"
)
if event.get("type") == "command.ack":
ack_seen = event.get("command_id") == lost_ack_command_id
break
result["checks"]["lost_ack_command_ack_emitted"] = ack_seen
if not ack_seen:
raise RuntimeError("server did not emit command.ack for the DDS status command")
result["checks"]["lost_ack_command_committed"] = await wait_for_command_checkpoint(
lost_ack_command_id,
)
if not result["checks"]["lost_ack_command_committed"]:
raise RuntimeError("station command ID did not reach the PostgreSQL checkpoint")
# Do not consume any frame after send: the browser's pending
# command remains unacknowledged when this transport is closed.
await lost_ack_socket.close()
result["lost_ack_command_id"] = lost_ack_command_id
result["lost_ack_ack_dispatched_to_browser"] = False
if args.failure_mode == "kill":
fault_service = owner_service
compose(args, "kill", fault_service)
owner_stopped = True
else:
proxy_name = "db-proxy-a" if row.backend_node_id == "backend-a" else "db-proxy-b"
fault_service = proxy_name
async with AsyncExitStack() as channels:
active_sockets = {}
for channel, path, cookie in (
("control", f"/ws/control/{session_id}", instructor_cookie),
("call", f"/ws/call/{session_id}", trainee_cookie),
("observe", f"/ws/observe/{session_id}", instructor_cookie),
("station", f"/ws/station/{session_id}", trainee_cookie),
):
active_sockets[channel] = await channels.enter_async_context(
ws_connect(ws_base, path, cookie)
)
compose(args, "stop", proxy_name)
fault_stopped = True
result["boundary_command_sent_to_old_owner"] = "arrived"
running_services = compose(args, "ps", "--status", "running", "--services").splitlines()
result["checks"]["owner_process_running"] = owner_service in running_services
if owner_service not in running_services:
raise RuntimeError("owner process did not remain running during DB partition")
closures = await asyncio.gather(*(
wait_for_fence(socket, timeout=args.takeover_timeout)
for socket in active_sockets.values()
))
result["existing_channel_closures"] = dict(zip(active_sockets, closures))
result["existing_channel_fence_notices"] = {
channel: closure["fence_notice"]
for channel, closure in result["existing_channel_closures"].items()
}
result["checks"].update({
f"existing_{channel}_closed": closure["closed"]
for channel, closure in result["existing_channel_closures"].items()
})
result["checks"].update({
f"existing_{channel}_fenced_with_1012": closure["fence_notice"]
for channel, closure in result["existing_channel_closures"].items()
})
deadline = time.monotonic() + args.takeover_timeout
takeover = None
while time.monotonic() < deadline:
candidate = await session_row()
if candidate and candidate.backend_node_id != row.backend_node_id:
takeover = candidate
break
await asyncio.sleep(0.5)
if takeover is None:
raise RuntimeError("other backend did not take over the expired lease")
result["takeover_owner"] = takeover.backend_node_id
result["takeover_epoch"] = takeover.backend_fencing_epoch
result["takeover_seconds"] = round(args.takeover_timeout - max(0, deadline - time.monotonic()), 2)
result["checks"]["owner_changed"] = takeover.backend_node_id != row.backend_node_id
result["checks"]["fencing_epoch_incremented"] = takeover.backend_fencing_epoch > old_epoch
result["checks"]["checkpoint_restored"] = bool(takeover.live_state and takeover.checkpoint_at)
result["checks"]["lost_ack_command_replayed"] = await replay_station_command(
ws_base, session_id, trainee_cookie, lost_ack_command,
lost_ack_command_id, attempts=args.reconnect_attempts,
)
if args.failure_mode == "partition":
compose(args, "start", fault_service)
fault_stopped = False
await asyncio.sleep(6) # let the surviving old process observe the newer epoch
old_backend_port = args.backend_port if owner_service == "backend" else args.backend_b_port
old_backend = f"http://127.0.0.1:{old_backend_port}"
try:
with urllib.request.urlopen(f"{old_backend}/api/health", timeout=5) as response:
result["checks"]["old_owner_process_healthy"] = response.status == 200
except Exception as exc:
result["checks"]["old_owner_process_healthy"] = False
result["old_owner_health_error"] = f"{type(exc).__name__}: {exc}"
stale = await probe_ws(
old_backend.replace("http://", "ws://"),
f"/ws/control/{session_id}", instructor_cookie,
)
result["stale_owner_control"] = stale
result["checks"]["stale_owner_control_rejected"] = not stale["connected"]
request = urllib.request.Request(
f"{args.frontend_url}/api/sessions/{session_id}",
headers={"Cookie": instructor_cookie},
)
rest_started = time.monotonic()
rest_deadline = rest_started + args.rest_timeout
rest_attempt = 0
while time.monotonic() < rest_deadline:
rest_attempt += 1
try:
with urllib.request.urlopen(
request,
timeout=min(3.0, max(0.2, rest_deadline - time.monotonic())),
context=ssl._create_unverified_context(),
) as response:
result["checks"]["session_rest"] = response.status == 200
if result["checks"]["session_rest"]:
result.pop("rest_error", None)
break
except urllib.error.HTTPError as exc:
result["rest_error"] = f"HTTP {exc.code}"
except Exception as exc:
result["rest_error"] = f"{type(exc).__name__}: {exc}"
await asyncio.sleep(min(1, max(0, rest_deadline - time.monotonic())))
result["checks"].setdefault("session_rest", False)
result["rest_attempts"] = rest_attempt
result["rest_elapsed_seconds"] = round(time.monotonic() - rest_started, 2)
async def reconnect_probe(path: str, cookie: str) -> dict:
last = {}
for attempt in range(1, args.reconnect_attempts + 1):
last = await probe_ws(ws_base, path, cookie)
if last["connected"]:
last["attempts"] = attempt
return last
await asyncio.sleep(1)
last["attempts"] = args.reconnect_attempts
return last
probes = {
"control": await reconnect_probe(f"/ws/control/{session_id}", instructor_cookie),
"call": await reconnect_probe(f"/ws/call/{session_id}", trainee_cookie),
"observe": await reconnect_probe(f"/ws/observe/{session_id}", instructor_cookie),
"station": await reconnect_probe(f"/ws/station/{session_id}", trainee_cookie),
}
result["websockets"] = probes
result["checks"].update({f"ws_{name}": probe["connected"] for name, probe in probes.items()})
# Reconcile the race status against the recovered checkpoint. If it
# committed before the fault, do not repeat it; otherwise issue it now.
recovered_snapshot = await station_command(ws_base, session_id, trainee_cookie)
recovered_status = recovered_snapshot["statuses"].get(service)
result["boundary_status_after_takeover"] = recovered_status
result["boundary_command_outcome"] = (
"committed before takeover" if recovered_status == "arrived"
else "not in recovered checkpoint; reconciled after takeover"
)
result["checks"]["boundary_command_reconciled"] = recovered_status in {
"responding", "arrived",
}
result["dds_progress_after_takeover"] = []
if recovered_status == "responding":
recovered_snapshot = await station_command(ws_base, session_id, trainee_cookie, {
"type": "card.status", "service": service, "status": "arrived",
"comment": "Основание: доклад по карточке.\n"
f"Сведения: бригада сообщила о прибытии; {service} notified.",
})
result["dds_progress_after_takeover"].append("arrived")
# Continue the same card after owner recovery and require a stored final
# score, not merely successful handshakes on the replacement owner.
for status, detail in (
("working", "crew started work"),
("completed", "crew completed work"),
):
snapshot = await station_command(ws_base, session_id, trainee_cookie, {
"type": "card.status", "service": service, "status": status,
"comment": f"Основание: доклад по карточке.\nСведения: {detail}; {service} notified.",
})
result["dds_progress_after_takeover"].append(status)
async with websockets.connect(
f"{ws_base}/ws/station/{session_id}",
additional_headers={"Cookie": trainee_cookie},
ssl=ssl._create_unverified_context() if ws_base.startswith("wss://") else None,
open_timeout=10, ping_interval=None,
) as socket:
await socket.send(json.dumps({"type": "station.finish"}))
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
raw = await asyncio.wait_for(socket.recv(), timeout=10)
event = json.loads(raw) if isinstance(raw, str) else {}
if event.get("type") == "error":
raise RuntimeError(f"station finish rejected: {event.get('message')}")
if event.get("type") == "score.ready":
result["checks"]["dds_finished"] = True
break
else:
raise RuntimeError("DDS exercise did not finish after takeover")
report_request = urllib.request.Request(
f"{args.frontend_url}/api/sessions/{session_id}/report",
headers={"Cookie": instructor_cookie},
)
with urllib.request.urlopen(
report_request, timeout=10, context=ssl._create_unverified_context(),
) as response:
report = json.loads(response.read())
result["dds_final_score"] = report.get("score_auto")
card_results = report.get("card_results", [])
if len(card_results) != 1:
raise RuntimeError(f"expected one DDS card result, got {len(card_results)}")
final_card = card_results[0]
result["dds_card_metrics"] = {
metric["key"]: metric["passed"] for metric in final_card["metrics"]
}
required_metric_keys = {
"dds_ack", "dds_decision", "dds_crew", "dds_progress",
"dds_completion", "dds_reply",
}
result["checks"]["dds_business_metrics_passed"] = all(
result["dds_card_metrics"].get(key) is True for key in required_metric_keys
)
from collections import Counter
observed_status_counts = Counter(
action.get("status") for action in final_card["actions"]
if action.get("type") == "card.status" and action.get("service") == service
)
expected_statuses = {"accepted", "responding", "arrived", "working", "completed"}
result["checks"]["dds_statuses_persisted_exactly_once"] = all(
observed_status_counts[status] == 1 for status in expected_statuses
)
result["dds_status_counts"] = dict(observed_status_counts)
result["checks"]["lost_ack_status_recorded_once"] = (
observed_status_counts["arrived"] == 1
)
passed = all(result["checks"].values())
result["pass"] = passed
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if passed else 1
finally:
if instructor_cookie:
try:
await send_control(ws_base, session_id, instructor_cookie, {"type": "session.stop"})
except Exception:
pass
if fault_stopped and fault_service:
compose(args, "start", fault_service, check=False)
if owner_stopped:
compose(args, "start", owner_service, check=False)
async with factory() as db:
row = await db.scalar(select(Session).where(Session.id == session_id))
if row:
await db.execute(delete(Session).where(Session.id == session_id))
await db.execute(delete(AuditLog).where(AuditLog.object_id == str(session_id)))
await db.execute(delete(AuditLog).where(AuditLog.actor.in_([instructor_login, trainee_login])))
await db.execute(delete(User).where(User.login.in_([instructor_login, trainee_login])))
await db.execute(delete(Trainee).where(Trainee.name == trainee.name))
await db.commit()
await engine.dispose()
await observer_engine.dispose()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--project", required=True)
parser.add_argument("--postgres-port", type=int, required=True)
parser.add_argument("--backend-port", type=int, required=True)
parser.add_argument("--backend-b-port", type=int, default=18001)
parser.add_argument("--frontend-port", type=int, required=True)
parser.add_argument("--tls-port", type=int, required=True)
parser.add_argument("--postgres-password", required=True)
parser.add_argument("--session-secret", required=True)
parser.add_argument("--tls-cert-dir", required=True)
parser.add_argument("--frontend-url", required=True)
parser.add_argument("--scenario", default="fire-apartment-l2")
parser.add_argument("--takeover-timeout", type=float, default=30)
parser.add_argument("--reconnect-attempts", type=int, default=5)
parser.add_argument("--rest-timeout", type=float, default=60)
parser.add_argument("--failure-mode", choices=("kill", "partition"), default="kill")
raise SystemExit(asyncio.run(main(parser.parse_args())))

402
scripts/test_cluster_scenario_registry.py Normal file
View file

@ -0,0 +1,402 @@
#!/usr/bin/env python3
"""Cross-process smoke: publish a trainee scenario on backend A, start it on B.
Use only with a disposable local Compose project and its own PostgreSQL. The
smoke creates temporary accounts, group, proposal, and DDS session and removes
their rows in ``finally``. It never writes evidence into the repository.
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
from pathlib import Path
import secrets
import ssl
import subprocess
import time
import urllib.error
import urllib.request
from uuid import UUID, uuid4
import websockets
def compose(args: argparse.Namespace, *command: str) -> str:
env = os.environ.copy()
env.update({
"POSTGRES_PORT": str(args.postgres_port),
"BACKEND_PORT": str(args.backend_a_port),
"BACKEND_B_PORT": str(args.backend_b_port),
"FRONTEND_PORT": str(args.frontend_port),
"TLS_PORT": str(args.tls_port),
"POSTGRES_PASSWORD": args.postgres_password,
"SESSION_SECRET": args.session_secret,
"TLS_CERT_DIR": args.tls_cert_dir,
})
files = [
"docker-compose.yml", "docker-compose.tls.yml",
"docker-compose.load-test.yml", "docker-compose.cluster.yml",
]
command_result = subprocess.run(
["docker", "compose", "-p", args.project,
*[part for file in files for part in ("-f", file)], *command],
check=False, capture_output=True, text=True, env=env,
)
if command_result.returncode:
raise RuntimeError(
f"docker compose {' '.join(command)} failed: {command_result.stderr.strip()}"
)
return command_result.stdout
def request_json(base: str, path: str, *, cookie: str | None = None,
payload: dict | None = None) -> dict | list:
headers = {"Content-Type": "application/json"}
if cookie:
headers["Cookie"] = cookie
data = json.dumps(payload, ensure_ascii=False).encode() if payload is not None else None
request = urllib.request.Request(f"{base}{path}", data=data, headers=headers,
method="POST" if data is not None else "GET")
with urllib.request.urlopen(request, timeout=15) as response:
return json.loads(response.read())
def login(base: str, login_name: str, password: str) -> str:
payload = json.dumps({"login": login_name, "password": password}).encode()
request = urllib.request.Request(
f"{base}/api/auth/login", data=payload, method="POST",
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(request, timeout=15) as response:
cookie = response.headers.get("Set-Cookie", "").split(";", 1)[0]
if not cookie.startswith("lct_session="):
raise RuntimeError(f"{base} did not issue the session cookie")
return cookie
async def main(args: argparse.Namespace) -> int:
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.api.auth import hash_password
from app.db.models import (
AuditLog, Group, Scenario as ScenarioRow, ScenarioSubmission,
Session, Trainee, User,
)
from app.domain import ekp
from app.scenarios import store
db_url = (f"postgresql+asyncpg://lct:{args.postgres_password}@127.0.0.1:"
f"{args.postgres_port}/lct")
engine = create_async_engine(db_url, pool_pre_ping=True)
factory = async_sessionmaker(engine, expire_on_commit=False)
suffix = uuid4().hex[:12]
teacher_login, trainee_login = f"registry-i-{suffix}", f"registry-t-{suffix}"
teacher_password, trainee_password = secrets.token_urlsafe(24), secrets.token_urlsafe(24)
teacher_name, trainee_name = "Registry smoke instructor", f"Registry smoke {suffix}"
group = Group(name=f"registry-smoke-{suffix}", owner_login=teacher_login)
trainee = Trainee(name=trainee_name)
scenario_id: str | None = None
session_id = uuid4()
session_ids: list[UUID] = []
submission_id: UUID | None = None
teacher_cookie = trainee_cookie = None
session_ws_base = None
ssl_context = None
result: dict = {"checks": {}}
a = f"http://127.0.0.1:{args.backend_a_port}"
b = f"http://127.0.0.1:{args.backend_b_port}"
try:
store.load_from_disk(Path(args.scenarios))
source = store.get(args.source_scenario)
if source is None or source.ground_truth.dds is None:
raise RuntimeError(f"invalid source scenario {args.source_scenario}")
trainee.group_id = None
async with factory() as db:
db.add(group)
await db.flush()
trainee.group_id = group.id
db.add(trainee)
await db.flush()
db.add_all([
User(login=teacher_login, full_name=teacher_name,
password_hash=hash_password(teacher_password), role="instructor",
blocked=False),
User(login=trainee_login, full_name=trainee_name,
password_hash=hash_password(trainee_password), role="trainee",
trainee_id=trainee.id, blocked=False),
])
await db.commit()
result["checks"]["backend_a_healthy"] = request_json(a, "/api/health").get("status") == "ok"
result["checks"]["backend_b_healthy"] = request_json(b, "/api/health").get("status") == "ok"
teacher_cookie = login(a, teacher_login, teacher_password)
trainee_cookie = login(a, trainee_login, trainee_password)
incident_group = ekp.incident(source.ground_truth.incident_code).group
created = request_json(a, "/api/scenario-submissions", cookie=trainee_cookie, payload={
"title": f"Межузловая проверка {suffix}", "level": source.level.value,
"kio": {
"caller_name": "Тестовый заявитель",
"caller_contact": "+7 900 000-00-00",
"address": f"Москва, учебная улица, дом {suffix[:3]}",
"description": "На учебном объекте обнаружено задымление и нужна бригада.",
"incident_group": incident_group,
"signs": source.signs,
"incident_type": source.type.value,
"dds": source.ground_truth.dds.value,
"victims_count": 0,
"fire": {"object_kind": "учебное помещение", "fire_nature": "задымление"},
},
})
submission_id = UUID(created["id"])
approved = request_json(a, f"/api/scenario-submissions/{submission_id}/review",
cookie=teacher_cookie,
payload={"decision": "approve", "comment": "Межузловой smoke."})
scenario_id = approved["scenario_id"]
result["scenario_id"] = scenario_id
result["published_on"] = "backend-a"
result["checks"]["submission_approved_on_a"] = approved["status"] == "approved"
# WS start is the first scenario operation on the selected target node.
# With --frontend-url, UUIDs are tried until Nginx consistent-hash routes
# one to B; every candidate is pinned for all session channels.
session_ws_base = args.frontend_url.replace("https://", "wss://").replace(
"http://", "ws://"
) if args.frontend_url else b.replace("http://", "ws://")
ssl_context = ssl._create_unverified_context() if session_ws_base.startswith("wss://") else None
target_backend = "backend-b"
max_route_attempts = 12 if args.frontend_url else 1
session_row = None
for route_attempt in range(max_route_attempts):
candidate_id = uuid4()
session_ids.append(candidate_id)
session_id = candidate_id
ws_url = f"{session_ws_base}/ws/control/{session_id}"
async with websockets.connect(
ws_url, additional_headers={"Cookie": teacher_cookie}, ssl=ssl_context,
open_timeout=15, ping_interval=None,
) as control:
await control.send(json.dumps({
"type": "scenario.start", "scenario_id": scenario_id,
"scenario_ids": [scenario_id], "trainee": trainee_name,
"trainee_id": str(trainee.id), "dds_service": created["kio"]["notify"][0],
"mode": "training", "exercise": "dds",
}, ensure_ascii=False))
async with factory() as db:
deadline = time.monotonic() + 8
session_row = None
while time.monotonic() < deadline:
session_row = await db.scalar(
select(Session).where(Session.id == candidate_id)
)
if session_row and session_row.backend_node_id:
break
await asyncio.sleep(0.1)
if session_row and session_row.backend_node_id == target_backend:
break
if session_row and session_row.backend_node_id:
result.setdefault("nginx_route_attempts", []).append({
"session_id": str(candidate_id),
"owner": session_row.backend_node_id,
})
await control.send(json.dumps({"type": "session.stop"}))
await asyncio.sleep(0.15)
else:
raise RuntimeError("session.start was not persisted through the selected route")
result["checks"]["session_started_on_backend_b"] = bool(
session_row and session_row.backend_node_id == target_backend
and session_row.scenario_id == scenario_id and session_row.live_state
)
if not result["checks"]["session_started_on_backend_b"]:
raise RuntimeError("could not route a fresh session to backend B")
result["session_id"] = str(session_id)
result["route_attempts"] = len(session_ids)
# Prove the user-visible catalog/detail routes while B is still alive.
catalog = request_json(b, "/api/scenarios", cookie=teacher_cookie)
entry = next((item for item in catalog if item["id"] == scenario_id), None)
result["checks"]["visible_in_backend_b_catalog"] = entry is not None
result["checks"]["owned_by_teacher_on_backend_b"] = bool(entry and entry["can_manage"])
detail = request_json(b, f"/api/scenarios/{scenario_id}", cookie=teacher_cookie)
result["checks"]["detail_loaded_from_backend_b"] = (
detail.get("student_card", {}).get("address")
== created["kio"]["address"]
)
if args.failure_mode == "kill-backend-b":
old_epoch = session_row.backend_fencing_epoch
compose(args, "kill", "backend-b")
result["checks"]["backend_b_killed_after_start"] = True
deadline = time.monotonic() + args.takeover_timeout
takeover_row = None
while time.monotonic() < deadline:
async with factory() as db:
takeover_row = await db.scalar(
select(Session).where(Session.id == session_id)
)
if takeover_row and takeover_row.backend_node_id == "backend-a":
break
await asyncio.sleep(0.25)
result["checks"]["takeover_to_backend_a"] = bool(
takeover_row and takeover_row.backend_node_id == "backend-a"
)
result["checks"]["fencing_epoch_incremented"] = bool(
takeover_row and takeover_row.backend_fencing_epoch > old_epoch
)
result["checks"]["checkpoint_restored_after_kill"] = bool(
takeover_row and takeover_row.live_state and takeover_row.checkpoint_at
)
if not all(result["checks"][key] for key in (
"takeover_to_backend_a", "fencing_epoch_incremented",
"checkpoint_restored_after_kill",
)):
raise RuntimeError("backend A did not take over the killed B session")
result["takeover_seconds"] = round(
args.takeover_timeout - max(0, deadline - time.monotonic()), 2
)
station_url = f"{session_ws_base}/ws/station/{session_id}"
station_owner_key = (
"dds_card_received_after_takeover" if args.failure_mode == "kill-backend-b"
else "dds_card_received_on_backend_b"
)
station_snapshot = None
received_card = None
last_station_error = None
for reconnect_attempt in range(8):
try:
async with websockets.connect(
station_url, additional_headers={"Cookie": trainee_cookie}, ssl=ssl_context,
open_timeout=10, ping_interval=None,
) as station:
for _ in range(60):
event = json.loads(await asyncio.wait_for(station.recv(), timeout=10))
if event.get("type") == "error":
raise RuntimeError(
f"station error after route/failover: {event.get('message')}"
)
if event.get("type") == "card.received":
received_card = event["card"]
elif event.get("type") == "station.state":
station_snapshot = event.get("snapshot")
if received_card is not None and station_snapshot is not None:
break
if received_card is None or station_snapshot is None:
raise RuntimeError("station reconnect did not restore card and snapshot")
break
except (TimeoutError, OSError, websockets.exceptions.WebSocketException,
RuntimeError) as exc:
last_station_error = exc
if reconnect_attempt == 7:
raise RuntimeError(f"station did not recover through proxy: {exc}") from exc
await asyncio.sleep(1)
result["checks"][station_owner_key] = received_card is not None
result["checks"]["approved_kio_preserved"] = bool(
received_card
and received_card.get("address") == created["kio"]["address"]
and received_card.get("caller_name") == created["kio"]["caller_name"]
)
if args.failure_mode == "kill-backend-b":
command_id = str(uuid4())
command = {
"type": "card.status", "service": station_snapshot["managed_service"],
"status": "accepted",
"comment": (
"Основание: доклад по карточке.\n"
f"Сведения: карточка повторно принята после takeover; {station_snapshot['managed_service']} notified."
),
"_command_id": command_id,
}
async with websockets.connect(
station_url, additional_headers={"Cookie": trainee_cookie}, ssl=ssl_context,
open_timeout=10, ping_interval=None,
) as station:
await station.send(json.dumps(command, ensure_ascii=False))
acked = False
for _ in range(40):
event = json.loads(await asyncio.wait_for(station.recv(), timeout=10))
if event.get("type") == "error":
raise RuntimeError(f"station command failed after takeover: {event.get('message')}")
if event.get("type") == "command.ack" and event.get("command_id") == command_id:
acked = True
break
result["checks"]["station_command_acked_after_takeover"] = acked
async with factory() as db:
restored = await db.scalar(select(Session).where(Session.id == session_id))
result["checks"]["post_takeover_command_checkpointed"] = bool(
restored and restored.backend_node_id == "backend-a"
and command_id in (restored.live_state or {}).get(
"processed_station_commands", []
)
)
result["pass"] = all(result["checks"].values())
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result["pass"] else 1
finally:
if args.failure_mode == "kill-backend-b" and args.project:
try:
compose(args, "start", "backend-b")
except Exception:
pass
if teacher_cookie and session_ws_base:
try:
stop_base = (
f"ws://127.0.0.1:{args.backend_a_port}"
if args.failure_mode == "kill-backend-b" else session_ws_base
)
stop_ssl = None if stop_base.startswith("ws://") else ssl_context
async with websockets.connect(
f"{stop_base}/ws/control/{session_id}",
additional_headers={"Cookie": teacher_cookie}, ssl=stop_ssl,
open_timeout=5, ping_interval=None,
) as control:
await control.send(json.dumps({"type": "session.stop"}))
except Exception:
pass
async with factory() as db:
await db.execute(delete(AuditLog).where(
AuditLog.object_id.in_([str(item) for item in session_ids])
))
if submission_id:
await db.execute(delete(ScenarioSubmission).where(
ScenarioSubmission.id == submission_id
))
if session_ids:
await db.execute(delete(Session).where(Session.id.in_(session_ids)))
if scenario_id:
await db.execute(delete(ScenarioRow).where(ScenarioRow.id == scenario_id))
await db.execute(delete(AuditLog).where(
AuditLog.actor.in_([teacher_login, trainee_login])
))
await db.execute(delete(User).where(User.login.in_([teacher_login, trainee_login])))
await db.execute(delete(Trainee).where(Trainee.name == trainee_name))
await db.execute(delete(Group).where(Group.id == group.id))
await db.commit()
await engine.dispose()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--postgres-port", type=int, required=True)
parser.add_argument("--backend-a-port", type=int, required=True)
parser.add_argument("--backend-b-port", type=int, required=True)
parser.add_argument("--postgres-password", required=True)
parser.add_argument("--frontend-url", help="optional TLS Nginx URL; tests consistent-hash routing")
parser.add_argument("--failure-mode", choices=("none", "kill-backend-b"), default="none")
parser.add_argument("--project", help="Compose project, required for --failure-mode kill-backend-b")
parser.add_argument("--tls-cert-dir", default="")
parser.add_argument("--session-secret", default="")
parser.add_argument("--frontend-port", type=int, default=15180)
parser.add_argument("--tls-port", type=int, default=15443)
parser.add_argument("--takeover-timeout", type=float, default=45)
parser.add_argument("--source-scenario", default="t01-1-fire-container")
parser.add_argument("--scenarios", default="scenarios")
parsed = parser.parse_args()
if parsed.failure_mode == "kill-backend-b" and not (
parsed.frontend_url and parsed.project and parsed.session_secret and parsed.tls_cert_dir
):
parser.error("kill-backend-b requires --frontend-url, --project, --session-secret and --tls-cert-dir")
raise SystemExit(asyncio.run(main(parsed)))

35
scripts/test_isolated_db.sh Normal file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
project="lct-test-$$-$RANDOM$RANDOM"
compose=(docker compose -p "$project" -f "$repo_root/docker-compose.test-db.yml")
cleanup() {
trap - EXIT INT TERM
"${compose[@]}" down --volumes --remove-orphans >/dev/null || true
}
trap cleanup EXIT
printf 'Изолированный тестовый Compose project: %s\n' "$project"
"${compose[@]}" up --pull never --detach --wait
port_mapping=$("${compose[@]}" port postgres 5432)
db_port="${port_mapping##*:}"
database_url="postgresql+asyncpg://lct_test:lct_test@127.0.0.1:${db_port}/lct_test"
evidence_dir="${LCT_TEST_EVIDENCE_DIR:-$repo_root/.local/evidence}"
mkdir -p "$evidence_dir"
cd "$repo_root/backend"
DATABASE_URL="$database_url" uv run --extra dev alembic upgrade head
DATABASE_URL="$database_url" uv run --extra dev python scripts/seed.py
DATABASE_URL="$database_url" uv run --extra dev python "$repo_root/scripts/load_db.py" \
--operations 1000 --workload audit --allow-audit-writes \
--output "$evidence_dir/db-audit-write-load-pool-2026-09-25.json"
DATABASE_URL="$database_url" uv run --extra dev python "$repo_root/scripts/load_db.py" \
--operations 1000 --concurrency 20 --workload audit --allow-audit-writes \
--output "$evidence_dir/db-audit-write-load-concurrent-2026-09-25.json"
if (($#)); then
DATABASE_URL="$database_url" uv run --extra dev pytest -q --tb=short --show-capture=no "$@"
else
DATABASE_URL="$database_url" uv run --extra dev pytest -q --tb=short --show-capture=no
fi

35
scripts/test_production_postgres.sh Normal file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
project="lct-production-check-$$"
port=$((20000 + $$ % 40000))
secret="prod-check-$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')"
compose=(docker compose -p "$project" -f "$repo_root/docker-compose.yml" \
-f "$repo_root/docker-compose.production.yml")
cleanup() {
POSTGRES_PASSWORD="$secret" POSTGRES_PORT="$port" \
"${compose[@]}" down --volumes --remove-orphans >/dev/null 2>&1 || true
}
trap cleanup EXIT INT TERM
if env -u POSTGRES_PASSWORD "${compose[@]}" config --quiet >/dev/null 2>&1; then
echo "ОШИБКА: production Compose запустился без POSTGRES_PASSWORD" >&2
exit 1
fi
POSTGRES_PASSWORD="$secret" POSTGRES_PORT="$port" \
"${compose[@]}" up --pull never --detach --wait postgres
network="${project}_default"
docker run --pull never --rm --network "$network" -e "PGPASSWORD=$secret" \
postgres:16-alpine psql -h postgres -U lct -d lct -Atc 'select 1' | rg -x '1' >/dev/null
if docker run --pull never --rm --network "$network" -e PGPASSWORD=wrong \
postgres:16-alpine psql -h postgres -U lct -d lct -Atc 'select 1' >/dev/null 2>&1; then
echo "ОШИБКА: production PostgreSQL принял неверный пароль" >&2
exit 1
fi
echo "Production PostgreSQL принял пароль из backend-сети и отклонил неверный."

199
scripts/test_recovery.py
View file

@ -12,6 +12,7 @@ from uuid import uuid4
from playwright.async_api import async_playwright
from sqlalchemy import delete
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from load_browser import control, local_url, login_cookie
@ -67,23 +68,27 @@ async def cleanup_temp_accounts(
logins: list[str],
trainee_name: str,
session_id,
*,
remove_accounts: bool,
) -> None:
engine = create_async_engine(database_url)
try:
factory = async_sessionmaker(engine, expire_on_commit=False)
async with factory() as db:
await db.execute(delete(Session).where(Session.id == session_id))
await db.execute(delete(User).where(User.login.in_(logins)))
await db.execute(delete(Trainee).where(Trainee.name == trainee_name))
await db.execute(delete(AuditLog).where(AuditLog.actor.in_(logins)))
await db.execute(delete(AuditLog).where(AuditLog.object_id == str(session_id)))
if remove_accounts:
await db.execute(delete(AuditLog).where(AuditLog.actor.in_(logins)))
await db.execute(delete(User).where(User.login.in_(logins)))
await db.execute(delete(Trainee).where(Trainee.name == trainee_name))
await db.commit()
finally:
await engine.dispose()
async def compose_service(action: str, service: str) -> None:
async def compose_service(project: str, action: str, service: str) -> None:
process = await asyncio.create_subprocess_exec(
"docker", "compose", action, service,
"docker", "compose", "--project-name", project, action, service,
cwd=ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
@ -94,25 +99,13 @@ async def compose_service(action: str, service: str) -> None:
raise RuntimeError(f"docker compose {action} {service}: {detail}")
async def wait_service_healthy(service: str, timeout: float = 45) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
process = await asyncio.create_subprocess_exec(
"docker", "compose", "ps", service,
cwd=ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, _ = await process.communicate()
if process.returncode == 0 and b"(healthy)" in stdout:
return
await asyncio.sleep(0.5)
raise TimeoutError(f"контейнер {service} не стал healthy за {timeout} с")
async def run(args: argparse.Namespace) -> int:
frontend = local_url(args.frontend_url, {"https"})
backend = local_url(args.backend_url, {"http"})
if not args.compose_project.strip():
raise ValueError("--compose-project обязателен: recovery test останавливает Nginx")
if not 0 <= args.outage_seconds <= 30:
raise ValueError("--outage-seconds должен быть от 0 до 30")
ephemeral = not args.login and not args.password
if bool(args.login) != bool(args.password):
raise ValueError("login и password задаются вместе либо не задаются")
@ -120,7 +113,7 @@ async def run(args: argparse.Namespace) -> int:
instructor_password = args.password
trainee_login = args.trainee_login
trainee_password = args.trainee_password
trainee_name = "recovery-check"
trainee_name = ""
trainee_id = None
session_uuid = uuid4()
if ephemeral:
@ -138,8 +131,26 @@ async def run(args: argparse.Namespace) -> int:
trainee_password,
trainee_name,
)
elif not trainee_login or not trainee_password:
raise ValueError("для существующих аккаунтов нужны trainee-login и trainee-password")
else:
if not trainee_login or not trainee_password:
raise ValueError("для существующих аккаунтов нужны trainee-login и trainee-password")
engine = create_async_engine(args.database_url)
try:
async with async_sessionmaker(engine, expire_on_commit=False)() as db:
row = (await db.execute(
select(Trainee.id, Trainee.name)
.join(User, User.trainee_id == Trainee.id)
.where(
User.login == trainee_login,
User.role == "trainee",
User.blocked.is_(False),
)
)).one_or_none()
if row is None:
raise ValueError("учётка курсанта не найдена или не привязана к профилю")
trainee_id, trainee_name = row
finally:
await engine.dispose()
assert instructor_login and instructor_password and trainee_login and trainee_password
instructor_cookie = login_cookie(backend, instructor_login, instructor_password)
trainee_cookie = login_cookie(backend, trainee_login, trainee_password)
@ -148,7 +159,6 @@ async def run(args: argparse.Namespace) -> int:
session_id = str(session_uuid)
started = False
frontend_stopped = False
backend_paused = False
try:
await control(ws_base, session_id, instructor_cookie, {
"type": "scenario.start",
@ -176,6 +186,41 @@ async def run(args: argparse.Namespace) -> int:
page = await context.new_page()
errors: list[str] = []
page.on("pageerror", lambda error: errors.append(str(error)))
await page.add_init_script("""(() => {
window.__lctDropOperatorKioAck = false;
window.__lctOperatorAckDropped = false;
let owner = WebSocket.prototype;
let descriptor;
while (owner && !descriptor) {
descriptor = Object.getOwnPropertyDescriptor(owner, 'onmessage');
if (!descriptor) owner = Object.getPrototypeOf(owner);
}
if (!owner || !descriptor?.set || !descriptor?.get) return;
Object.defineProperty(owner, 'onmessage', {
configurable: descriptor.configurable,
enumerable: descriptor.enumerable,
get() { return descriptor.get.call(this); },
set(handler) {
const socket = this;
const wrapped = (event) => {
if (window.__lctDropOperatorKioAck && typeof event.data === 'string') {
try {
const message = JSON.parse(event.data);
if (message.type === 'kio.patch' && message.source === 'operator') {
window.__lctDropOperatorKioAck = false;
window.__lctOperatorAckDropped = true;
socket.close(4000, 'test: discard checkpoint acknowledgement');
return;
}
} catch { /* non-JSON frames are passed to the application */ }
}
if (handler) handler.call(socket, event);
};
descriptor.set.call(socket, handler ? wrapped : null);
},
});
window.__lctAckInterceptorInstalled = Boolean(descriptor?.set && descriptor?.get);
})()""")
await page.goto(
f"{frontend}/trainee?session={session_id}",
wait_until="domcontentloaded",
@ -185,6 +230,8 @@ async def run(args: argparse.Namespace) -> int:
await address.wait_for(
timeout=args.timeout * 1000,
)
if not await page.evaluate("window.__lctAckInterceptorInstalled"):
raise RuntimeError("browser could not install the WebSocket acknowledgement test hook")
await page.locator(".arm-topbar-state .state-ok").wait_for(
timeout=args.timeout * 1000,
)
@ -198,36 +245,70 @@ async def run(args: argparse.Namespace) -> int:
buffered_value = "улица Буферная, дом 30"
started_at = time.perf_counter()
# Держим существующее WSS-соединение открытым, но не даём backend
# подтвердить правку. Это воспроизводит пакет, зависший ровно в
# момент отказа прокси, без искусственного доступа к JS-сокету.
await compose_service("pause", "backend")
backend_paused = True
await address.fill(buffered_value)
await page.locator(".kio-arm-row-address.kio-arm-pending").wait_for(
timeout=args.timeout * 1000,
)
await compose_service("stop", "frontend")
# Останавливаем proxy и вводим значение только после того, как
# browser отметил канал отключённым: это проверяет offline outbox.
await compose_service(args.compose_project, "stop", "frontend")
frontend_stopped = True
outage_started_at = time.perf_counter()
await page.wait_for_function(
"window.__lctRecoveryStates.some(value => !value.includes('АРМ подключён'))",
timeout=args.timeout * 1000,
)
await compose_service("unpause", "backend")
backend_paused = False
await wait_service_healthy("backend")
await compose_service("start", "frontend")
# The operator edits only after the channel is known to be down.
# This exercises the disconnected outbox path instead of relying
# on a frame which might already have reached the server.
await address.fill(buffered_value)
await page.locator(".kio-arm-row-address.kio-arm-pending").wait_for(
timeout=args.timeout * 1000,
)
outage_remaining = args.outage_seconds - (time.perf_counter() - outage_started_at)
if outage_remaining > 0:
await asyncio.sleep(outage_remaining)
restore_started_at = time.perf_counter()
proxy_unavailable_seconds = restore_started_at - outage_started_at
await compose_service(args.compose_project, "start", "frontend")
frontend_stopped = False
await page.wait_for_function(
"window.__lctRecoveryStates.at(-1).includes('АРМ подключён')",
timeout=args.timeout * 1000,
)
recovery_seconds = time.perf_counter() - restore_started_at
await page.wait_for_function(
"!document.querySelector('.kio-arm-row-address')?.classList.contains('kio-arm-pending')",
timeout=args.timeout * 1000,
)
buffered_value_after = await page.locator(".kio-arm-row-address input").input_value()
# Now send while the WSS is open, allow PostgreSQL checkpoint/echo
# to complete, then deliberately discard that one echo and close
# the browser socket. This targets the ambiguous commit/ack window:
# reconnect must recover the stored value without duplicating a
# non-replace-style action or leaving the field pending.
committed_value = "улица Подтверждённая, дом 31"
await page.evaluate("window.__lctDropOperatorKioAck = true")
await address.fill(committed_value)
await page.wait_for_function(
"window.__lctOperatorAckDropped === true",
timeout=args.timeout * 1000,
)
await page.wait_for_function(
"window.__lctRecoveryStates.some(value => !value.includes('АРМ подключён'))",
timeout=args.timeout * 1000,
)
await page.wait_for_function(
"window.__lctRecoveryStates.at(-1).includes('АРМ подключён')",
timeout=args.timeout * 1000,
)
await page.wait_for_function(
"!document.querySelector('.kio-arm-row-address')?.classList.contains('kio-arm-pending')",
timeout=args.timeout * 1000,
)
buffered_value_after = await page.locator(".kio-arm-row-address input").input_value()
recovery_seconds = time.perf_counter() - started_at
committed_value_after = await page.locator(".kio-arm-row-address input").input_value()
ack_deliberately_dropped = await page.evaluate("window.__lctOperatorAckDropped")
pending_cleared = not await page.locator(".kio-arm-row-address").evaluate(
"element => element.classList.contains('kio-arm-pending')"
)
total_seconds = time.perf_counter() - started_at
states = await page.evaluate("window.__lctRecoveryStates")
await context.close()
await browser.close()
@ -237,19 +318,37 @@ async def run(args: argparse.Namespace) -> int:
"session_id": session_id,
"component_restarted": "frontend (Nginx TLS/WSS proxy) container stop/start",
"observed_states": states,
"recovery_seconds": recovery_seconds,
"requested_outage_seconds": args.outage_seconds,
"proxy_unavailable_seconds": proxy_unavailable_seconds,
"recovery_after_restore_seconds": recovery_seconds,
"total_seconds": total_seconds,
"browser_errors": errors,
"buffered_patch": {
"field": "address",
"value": buffered_value_after,
"confirmed_by_server": buffered_value_after == buffered_value,
},
"lost_ack_recovery": {
"ack_deliberately_dropped": ack_deliberately_dropped,
"value_after_reconnect": committed_value_after,
"confirmed_by_recovered_server_state": committed_value_after == committed_value,
"pending_cleared": pending_cleared,
},
"pass_recovery_30s": (
recovery_seconds <= 30 and not errors and buffered_value_after == buffered_value
args.outage_seconds <= 30
and proxy_unavailable_seconds <= 30.5
and recovery_seconds <= 30
and not errors
and buffered_value_after == buffered_value
and committed_value_after == committed_value
and ack_deliberately_dropped
and pending_cleared
),
"scope": (
"one live card exercise; backend acknowledgement is stalled after address edit; "
"Nginx is stopped; after recovery reconnect repeats only the idempotent KIO patch"
"one live card exercise; Nginx is unavailable while the trainee enters an address; "
"the disconnected KIO patch is queued and confirmed after WSS reconnect; a second "
"KIO patch is committed while connected, its checkpoint echo is deliberately "
"discarded, and reconnect recovers that state; backend remains running"
),
}
rendered = json.dumps(result, ensure_ascii=False, indent=2)
@ -259,11 +358,8 @@ async def run(args: argparse.Namespace) -> int:
stream.write(rendered + "\n")
return 0 if result["pass_recovery_30s"] else 1
finally:
if backend_paused:
await compose_service("unpause", "backend")
await wait_service_healthy("backend")
if frontend_stopped:
await compose_service("start", "frontend")
await compose_service(args.compose_project, "start", "frontend")
if started:
await control(ws_base, session_id, instructor_cookie, {"type": "session.stop"})
if ephemeral:
@ -272,6 +368,7 @@ async def run(args: argparse.Namespace) -> int:
[instructor_login, trainee_login],
trainee_name,
session_uuid,
remove_accounts=ephemeral,
)
@ -279,6 +376,8 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--frontend-url", default="https://127.0.0.1:5443")
parser.add_argument("--backend-url", default="http://127.0.0.1:8000")
parser.add_argument("--compose-project", required=True,
help="точное имя изолированного Compose project; тест останавливает его frontend")
parser.add_argument("--login")
parser.add_argument("--password")
parser.add_argument("--trainee-login")
@ -289,6 +388,8 @@ if __name__ == "__main__":
)
parser.add_argument("--scenario", default="fire-apartment-l2")
parser.add_argument("--timeout", type=float, default=30)
parser.add_argument("--outage-seconds", type=float, default=0,
help="держать frontend недоступным столько секунд (0–30), вводить карточку после разрыва")
parser.add_argument("--ignore-https-errors", action="store_true")
parser.add_argument("--output")
try:

48
scripts/test_sip_isolated.sh Normal file
View file

@ -0,0 +1,48 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
project="lct-sip-smoke-$(date +%Y%m%d)-$$-${RANDOM:-0}"
compose=(docker compose -p "$project" -f "$repo_root/docker-compose.yml" -f "$repo_root/docker-compose.sip.yml")
sip_port="${SIP_SMOKE_PORT:-15170}"
sips_port="${SIP_SMOKE_SIPS_PORT:-15171}"
websocket_port="${SIP_SMOKE_WS_PORT:-18098}"
rtp_start="${SIP_SMOKE_RTP_START:-12000}"
rtp_end=$((rtp_start + 99))
rtp_offset=$((rtp_start - 10000))
tls_dir="$(mktemp -d /tmp/lct-sip-smoke-tls-XXXXXX)"
sip_smoke_password="$(openssl rand -hex 24)"
cleanup() {
trap - EXIT INT TERM
"${compose[@]}" down --volumes --remove-orphans >/dev/null || true
case "$tls_dir" in
/tmp/lct-sip-smoke-tls-*) rm -rf -- "$tls_dir" ;;
*) printf 'Refusing to remove unexpected TLS temp path: %s\n' "$tls_dir" >&2 ;;
esac
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
python3 -c 'import socket,sys; ports=[("tcp",int(sys.argv[1])),("udp",int(sys.argv[1])),("tcp",int(sys.argv[2])),("tcp",int(sys.argv[3]))]+[("udp",p) for p in range(int(sys.argv[4]),int(sys.argv[5])+1)]; sockets=[]
try:
for kind,port in ports:
sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM if kind=="tcp" else socket.SOCK_DGRAM)
sock.bind(("127.0.0.1",port)); sockets.append(sock)
except OSError as exc:
raise SystemExit(f"SIP smoke port {port}/{kind} is unavailable: {exc}")
finally:
[sock.close() for sock in sockets]' "$sip_port" "$sips_port" "$websocket_port" "$rtp_start" "$rtp_end"
printf 'Starting isolated SIP smoke project: %s\n' "$project"
SIP_6001_PASSWORD="$sip_smoke_password" \
SIP_PORT="$sip_port" SIPS_PORT="$sips_port" SIP_WS_PORT="$websocket_port" \
RTP_PORT_START="$rtp_start" RTP_PORT_END="$rtp_end" \
SIP_TLS_DIR="$tls_dir" SIP_EXTERNAL_MEDIA_ADDRESS=127.0.0.1 \
"${compose[@]}" up --pull never --no-build --detach --wait --no-deps sip
SIP_PASSWORD="$sip_smoke_password" python3 "$repo_root/scripts/smoke_sip.py" \
--host 127.0.0.1 --port "$sip_port" --rtp-host-offset "$rtp_offset" \
--output "$repo_root/docs/evidence/sip-rtp-2026-09-25.json"

83
scripts/test_webrtc_isolated.sh Normal file
View file

@ -0,0 +1,83 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
project="lct-webrtc-smoke-$$-${RANDOM:-0}"
compose=(docker compose -p "$project" \
-f "$repo_root/docker-compose.yml" \
-f "$repo_root/docker-compose.tls.yml" \
-f "$repo_root/docker-compose.sip.yml" \
-f "$repo_root/docker-compose.webrtc-test.yml")
temp_root="$(mktemp -d /tmp/lct-webrtc-smoke-XXXXXX)"
# Keep all ports in one checked block; RTP gets its own 100-port block.
base_port=$((30000 + ($$ % 15000)))
export COMPOSE_PROJECT_NAME="$project"
export BIND_HOST=127.0.0.1
export POSTGRES_PORT=$base_port
export BACKEND_PORT=$((base_port + 1))
export FRONTEND_PORT=$((base_port + 2))
export TLS_PORT=$((base_port + 3))
export SIP_PORT=$((base_port + 4))
export SIPS_PORT=$((base_port + 5))
export SIP_WS_PORT=$((base_port + 6))
export RTP_PORT_START=$((base_port + 100))
export RTP_PORT_END=$((base_port + 199))
export SIP_RTP_START="$RTP_PORT_START"
export SIP_RTP_END="$RTP_PORT_END"
export POSTGRES_PASSWORD="$(openssl rand -hex 24)"
export SESSION_SECRET="$(openssl rand -hex 48)"
export TLS_CERT_DIR="$temp_root/frontend-tls"
export SIP_TLS_DIR="$temp_root/sip-tls"
export SIP_EXTERNAL_MEDIA_ADDRESS=127.0.0.1
mkdir -p "$TLS_CERT_DIR" "$SIP_TLS_DIR"
cleanup() {
status=$?
trap - EXIT INT TERM
if ((status != 0)); then
failure_log="/tmp/${project}-compose.log"
"${compose[@]}" logs --no-color --tail 250 >"$failure_log" 2>&1 || true
printf 'Isolated WebRTC diagnostics saved to %s\n' "$failure_log" >&2
fi
"${compose[@]}" down --volumes --remove-orphans >/dev/null || true
case "$temp_root" in
/tmp/lct-webrtc-smoke-*) rm -rf -- "$temp_root" ;;
*) printf 'Refusing to remove unexpected temporary path: %s\n' "$temp_root" >&2 ;;
esac
exit "$status"
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
python3 -c 'import socket,sys
ports=[("tcp",int(sys.argv[1])),("tcp",int(sys.argv[2])),("tcp",int(sys.argv[3])),
("tcp",int(sys.argv[4])),("tcp",int(sys.argv[5])),("udp",int(sys.argv[5])),
("tcp",int(sys.argv[6])),("tcp",int(sys.argv[7]))]
ports.extend(("udp",p) for p in range(int(sys.argv[8]),int(sys.argv[9])+1))
sockets=[]
try:
for kind,port in ports:
sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM if kind=="tcp" else socket.SOCK_DGRAM)
sock.bind(("127.0.0.1",port)); sockets.append(sock)
except OSError as exc:
raise SystemExit(f"isolated WebRTC smoke port {port}/{kind} unavailable: {exc}")
finally:
[sock.close() for sock in sockets]' \
"$POSTGRES_PORT" "$BACKEND_PORT" "$FRONTEND_PORT" "$TLS_PORT" "$SIP_PORT" \
"$SIPS_PORT" "$SIP_WS_PORT" "$RTP_PORT_START" "$RTP_PORT_END"
printf 'Starting isolated WebRTC smoke project: %s\n' "$project"
cd "$repo_root"
npm --prefix frontend run build
"${compose[@]}" build backend sip
"${compose[@]}" up --no-build --pull never --detach --wait --wait-timeout 300 \
postgres backend frontend sip
python3 "$repo_root/scripts/smoke_webrtc_browser.py" \
--frontend-url "https://127.0.0.1:$TLS_PORT" \
--backend-url "http://127.0.0.1:$BACKEND_PORT" \
--database-url "postgresql+asyncpg://lct:${POSTGRES_PASSWORD}@127.0.0.1:${POSTGRES_PORT}/lct" \
--timeout "${WEBRTC_TEST_TIMEOUT:-60}" \
--output "$repo_root/docs/evidence/webrtc-browser-isolated-2026-09-26.json"

25
scripts/validate_production_env.sh Normal file
View file

@ -0,0 +1,25 @@
#!/usr/bin/env bash
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
if [[ -n "${POSTGRES_PASSWORD+x}" ]]; then
password="$POSTGRES_PASSWORD"
elif [[ -f "$repo_root/.env" ]]; then
matches="$(grep -c '^POSTGRES_PASSWORD=' "$repo_root/.env" || true)"
if [[ "$matches" != "1" ]]; then
echo "Укажите ровно один POSTGRES_PASSWORD в .env или окружении." >&2
exit 2
fi
password="$(sed -n 's/^POSTGRES_PASSWORD=//p' "$repo_root/.env")"
else
echo "Для production запуска скопируйте .env.example в .env и задайте POSTGRES_PASSWORD." >&2
exit 2
fi
if [[ ! "$password" =~ ^[A-Za-z0-9._~-]{32,}$ ]]; then
echo "POSTGRES_PASSWORD должен содержать не менее 32 символов: латиница, цифры, точка, дефис, подчёркивание или тильда. Значение не показано." >&2
exit 2
fi
echo "Production DB secret задан и удовлетворяет формату (само значение не отображается)."