298 lines
12 KiB
Python
298 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""Замер переподключения браузерного WSS после перезапуска frontend-proxy."""
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import secrets
|
||
import sys
|
||
import time
|
||
from pathlib import Path
|
||
from uuid import uuid4
|
||
|
||
from playwright.async_api import async_playwright
|
||
from sqlalchemy import delete
|
||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||
|
||
from load_browser import control, local_url, login_cookie
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / "backend"))
|
||
|
||
from app.api.auth import hash_password # noqa: E402
|
||
from app.db.models import AuditLog, Session, Trainee, User # noqa: E402
|
||
|
||
|
||
async def create_temp_accounts(
|
||
database_url: str,
|
||
instructor_login: str,
|
||
instructor_password: str,
|
||
trainee_login: str,
|
||
trainee_password: str,
|
||
trainee_name: str,
|
||
):
|
||
"""Создать изолированную пару ролей и вернуть ID привязанного курсанта."""
|
||
engine = create_async_engine(database_url)
|
||
try:
|
||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||
async with factory() as db:
|
||
trainee = Trainee(name=trainee_name)
|
||
db.add(trainee)
|
||
await db.flush()
|
||
db.add_all([
|
||
User(
|
||
login=instructor_login,
|
||
full_name="Преподаватель проверки восстановления",
|
||
password_hash=hash_password(instructor_password),
|
||
role="instructor",
|
||
blocked=False,
|
||
),
|
||
User(
|
||
login=trainee_login,
|
||
full_name=trainee_name,
|
||
password_hash=hash_password(trainee_password),
|
||
role="trainee",
|
||
trainee_id=trainee.id,
|
||
blocked=False,
|
||
),
|
||
])
|
||
await db.commit()
|
||
return trainee.id
|
||
finally:
|
||
await engine.dispose()
|
||
|
||
|
||
async def cleanup_temp_accounts(
|
||
database_url: str,
|
||
logins: list[str],
|
||
trainee_name: str,
|
||
session_id,
|
||
) -> None:
|
||
engine = create_async_engine(database_url)
|
||
try:
|
||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||
async with factory() as db:
|
||
await db.execute(delete(Session).where(Session.id == session_id))
|
||
await db.execute(delete(User).where(User.login.in_(logins)))
|
||
await db.execute(delete(Trainee).where(Trainee.name == trainee_name))
|
||
await db.execute(delete(AuditLog).where(AuditLog.actor.in_(logins)))
|
||
await db.commit()
|
||
finally:
|
||
await engine.dispose()
|
||
|
||
|
||
async def compose_service(action: str, service: str) -> None:
|
||
process = await asyncio.create_subprocess_exec(
|
||
"docker", "compose", action, service,
|
||
cwd=ROOT,
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.PIPE,
|
||
)
|
||
stdout, stderr = await process.communicate()
|
||
if process.returncode:
|
||
detail = (stderr or stdout).decode(errors="replace").strip()
|
||
raise RuntimeError(f"docker compose {action} {service}: {detail}")
|
||
|
||
|
||
async def wait_service_healthy(service: str, timeout: float = 45) -> None:
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
process = await asyncio.create_subprocess_exec(
|
||
"docker", "compose", "ps", service,
|
||
cwd=ROOT,
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.PIPE,
|
||
)
|
||
stdout, _ = await process.communicate()
|
||
if process.returncode == 0 and b"(healthy)" in stdout:
|
||
return
|
||
await asyncio.sleep(0.5)
|
||
raise TimeoutError(f"контейнер {service} не стал healthy за {timeout} с")
|
||
|
||
|
||
async def run(args: argparse.Namespace) -> int:
|
||
frontend = local_url(args.frontend_url, {"https"})
|
||
backend = local_url(args.backend_url, {"http"})
|
||
ephemeral = not args.login and not args.password
|
||
if bool(args.login) != bool(args.password):
|
||
raise ValueError("login и password задаются вместе либо не задаются")
|
||
instructor_login = args.login
|
||
instructor_password = args.password
|
||
trainee_login = args.trainee_login
|
||
trainee_password = args.trainee_password
|
||
trainee_name = "recovery-check"
|
||
trainee_id = None
|
||
session_uuid = uuid4()
|
||
if ephemeral:
|
||
suffix = session_uuid.hex[:10]
|
||
instructor_login = f"recovery-instructor-{suffix}"
|
||
instructor_password = secrets.token_urlsafe(24)
|
||
trainee_login = f"recovery-trainee-{suffix}"
|
||
trainee_password = secrets.token_urlsafe(24)
|
||
trainee_name = f"recovery-trainee-{suffix}"
|
||
trainee_id = await create_temp_accounts(
|
||
args.database_url,
|
||
instructor_login,
|
||
instructor_password,
|
||
trainee_login,
|
||
trainee_password,
|
||
trainee_name,
|
||
)
|
||
elif not trainee_login or not trainee_password:
|
||
raise ValueError("для существующих аккаунтов нужны trainee-login и trainee-password")
|
||
assert instructor_login and instructor_password and trainee_login and trainee_password
|
||
instructor_cookie = login_cookie(backend, instructor_login, instructor_password)
|
||
trainee_cookie = login_cookie(backend, trainee_login, trainee_password)
|
||
cookie_name, cookie_value = trainee_cookie.split("=", 1)
|
||
ws_base = "ws" + backend.removeprefix("http")
|
||
session_id = str(session_uuid)
|
||
started = False
|
||
frontend_stopped = False
|
||
backend_paused = False
|
||
try:
|
||
await control(ws_base, session_id, instructor_cookie, {
|
||
"type": "scenario.start",
|
||
"scenario_id": args.scenario,
|
||
"trainee": trainee_name,
|
||
"trainee_id": str(trainee_id) if trainee_id else None,
|
||
"mode": "training",
|
||
"exercise": "card",
|
||
})
|
||
started = True
|
||
async with async_playwright() as playwright:
|
||
browser = await playwright.chromium.launch(headless=True)
|
||
context = await browser.new_context(
|
||
viewport={"width": 1280, "height": 720},
|
||
ignore_https_errors=args.ignore_https_errors,
|
||
)
|
||
await context.add_cookies([{
|
||
"name": cookie_name,
|
||
"value": cookie_value,
|
||
"url": frontend,
|
||
"httpOnly": True,
|
||
"secure": True,
|
||
"sameSite": "Lax",
|
||
}])
|
||
page = await context.new_page()
|
||
errors: list[str] = []
|
||
page.on("pageerror", lambda error: errors.append(str(error)))
|
||
await page.goto(
|
||
f"{frontend}/trainee?session={session_id}",
|
||
wait_until="domcontentloaded",
|
||
timeout=args.timeout * 1000,
|
||
)
|
||
address = page.locator(".kio-arm-row-address input")
|
||
await address.wait_for(
|
||
timeout=args.timeout * 1000,
|
||
)
|
||
await page.locator(".arm-topbar-state .state-ok").wait_for(
|
||
timeout=args.timeout * 1000,
|
||
)
|
||
await page.evaluate("""() => {
|
||
const node = document.querySelector('.arm-topbar-state');
|
||
window.__lctRecoveryStates = [node?.textContent ?? 'missing'];
|
||
new MutationObserver(() => window.__lctRecoveryStates.push(
|
||
node?.textContent ?? 'missing'
|
||
)).observe(node, {attributes: true, childList: true, subtree: true});
|
||
}""")
|
||
|
||
buffered_value = "улица Буферная, дом 30"
|
||
started_at = time.perf_counter()
|
||
# Держим существующее WSS-соединение открытым, но не даём backend
|
||
# подтвердить правку. Это воспроизводит пакет, зависший ровно в
|
||
# момент отказа прокси, без искусственного доступа к JS-сокету.
|
||
await compose_service("pause", "backend")
|
||
backend_paused = True
|
||
await address.fill(buffered_value)
|
||
await page.locator(".kio-arm-row-address.kio-arm-pending").wait_for(
|
||
timeout=args.timeout * 1000,
|
||
)
|
||
await compose_service("stop", "frontend")
|
||
frontend_stopped = True
|
||
await page.wait_for_function(
|
||
"window.__lctRecoveryStates.some(value => !value.includes('АРМ подключён'))",
|
||
timeout=args.timeout * 1000,
|
||
)
|
||
await compose_service("unpause", "backend")
|
||
backend_paused = False
|
||
await wait_service_healthy("backend")
|
||
await compose_service("start", "frontend")
|
||
frontend_stopped = False
|
||
await page.wait_for_function(
|
||
"window.__lctRecoveryStates.at(-1).includes('АРМ подключён')",
|
||
timeout=args.timeout * 1000,
|
||
)
|
||
await page.wait_for_function(
|
||
"!document.querySelector('.kio-arm-row-address')?.classList.contains('kio-arm-pending')",
|
||
timeout=args.timeout * 1000,
|
||
)
|
||
buffered_value_after = await page.locator(".kio-arm-row-address input").input_value()
|
||
recovery_seconds = time.perf_counter() - started_at
|
||
states = await page.evaluate("window.__lctRecoveryStates")
|
||
await context.close()
|
||
await browser.close()
|
||
|
||
result = {
|
||
"frontend": frontend,
|
||
"session_id": session_id,
|
||
"component_restarted": "frontend (Nginx TLS/WSS proxy) container stop/start",
|
||
"observed_states": states,
|
||
"recovery_seconds": recovery_seconds,
|
||
"browser_errors": errors,
|
||
"buffered_patch": {
|
||
"field": "address",
|
||
"value": buffered_value_after,
|
||
"confirmed_by_server": buffered_value_after == buffered_value,
|
||
},
|
||
"pass_recovery_30s": (
|
||
recovery_seconds <= 30 and not errors and buffered_value_after == buffered_value
|
||
),
|
||
"scope": (
|
||
"one live card exercise; backend acknowledgement is stalled after address edit; "
|
||
"Nginx is stopped; after recovery reconnect repeats only the idempotent KIO patch"
|
||
),
|
||
}
|
||
rendered = json.dumps(result, ensure_ascii=False, indent=2)
|
||
print(rendered)
|
||
if args.output:
|
||
with open(args.output, "w", encoding="utf-8") as stream:
|
||
stream.write(rendered + "\n")
|
||
return 0 if result["pass_recovery_30s"] else 1
|
||
finally:
|
||
if backend_paused:
|
||
await compose_service("unpause", "backend")
|
||
await wait_service_healthy("backend")
|
||
if frontend_stopped:
|
||
await compose_service("start", "frontend")
|
||
if started:
|
||
await control(ws_base, session_id, instructor_cookie, {"type": "session.stop"})
|
||
if ephemeral:
|
||
await cleanup_temp_accounts(
|
||
args.database_url,
|
||
[instructor_login, trainee_login],
|
||
trainee_name,
|
||
session_uuid,
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--frontend-url", default="https://127.0.0.1:5443")
|
||
parser.add_argument("--backend-url", default="http://127.0.0.1:8000")
|
||
parser.add_argument("--login")
|
||
parser.add_argument("--password")
|
||
parser.add_argument("--trainee-login")
|
||
parser.add_argument("--trainee-password")
|
||
parser.add_argument(
|
||
"--database-url",
|
||
default="postgresql+asyncpg://lct:lct@127.0.0.1:5432/lct",
|
||
)
|
||
parser.add_argument("--scenario", default="fire-apartment-l2")
|
||
parser.add_argument("--timeout", type=float, default=30)
|
||
parser.add_argument("--ignore-https-errors", action="store_true")
|
||
parser.add_argument("--output")
|
||
try:
|
||
raise SystemExit(asyncio.run(run(parser.parse_args())))
|
||
except Exception as exc:
|
||
print(f"recovery check failed: {type(exc).__name__}: {exc}")
|
||
raise SystemExit(2)
|