Complete training workflow and acceptance hardening
This commit is contained in:
andreysk0304
gglamer
parent
4c4b91064f
commit
7237265833
243 changed files with 17014 additions and 1500 deletions
|
|
@ -6,15 +6,196 @@ import asyncio
|
|||
import inspect
|
||||
import json
|
||||
import math
|
||||
import secrets
|
||||
import ssl
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import websockets
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "backend"))
|
||||
|
||||
|
||||
async def create_distinct_users(
|
||||
database_url: str, sessions: int, observers: int, *, create_controller: bool
|
||||
) -> dict:
|
||||
"""Provision short-lived local accounts for a literal 100-user load test."""
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.api.auth import hash_password
|
||||
from app.db.models import Trainee, User
|
||||
|
||||
db_host = make_url(database_url).host
|
||||
if db_host not in {"127.0.0.1", "localhost", "::1"}:
|
||||
raise ValueError("--distinct-users разрешён только для loopback PostgreSQL")
|
||||
|
||||
suffix = uuid4().hex[:12]
|
||||
engine = create_async_engine(database_url)
|
||||
users: list[dict] = []
|
||||
trainee_ids: list[str] = []
|
||||
trainee_names: list[str] = []
|
||||
controller = None
|
||||
try:
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
if create_controller:
|
||||
controller = {
|
||||
"login": f"load-i-{suffix}",
|
||||
"password": secrets.token_urlsafe(24),
|
||||
}
|
||||
db.add(User(
|
||||
login=controller["login"],
|
||||
full_name=f"Нагрузочный преподаватель {suffix}",
|
||||
password_hash=hash_password(controller["password"]),
|
||||
role="instructor",
|
||||
blocked=False,
|
||||
))
|
||||
for index in range(sessions):
|
||||
name = f"Нагрузочный курсант {suffix}-{index:02d}"
|
||||
login = f"load-t-{suffix}-{index:02d}"
|
||||
password = secrets.token_urlsafe(24)
|
||||
trainee = Trainee(name=name)
|
||||
db.add(trainee)
|
||||
await db.flush()
|
||||
db.add(User(
|
||||
login=login,
|
||||
full_name=name,
|
||||
password_hash=hash_password(password),
|
||||
role="trainee",
|
||||
trainee_id=trainee.id,
|
||||
blocked=False,
|
||||
))
|
||||
users.append({"login": login, "password": password})
|
||||
trainee_ids.append(str(trainee.id))
|
||||
trainee_names.append(name)
|
||||
for index in range(observers):
|
||||
login = f"load-a-{suffix}-{index:03d}"
|
||||
password = secrets.token_urlsafe(24)
|
||||
db.add(User(
|
||||
login=login,
|
||||
full_name=f"Нагрузочный наблюдатель {suffix}-{index:03d}",
|
||||
password_hash=hash_password(password),
|
||||
role="admin",
|
||||
blocked=False,
|
||||
))
|
||||
users.append({"login": login, "password": password})
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await engine.dispose()
|
||||
raise
|
||||
await engine.dispose()
|
||||
return {
|
||||
"suffix": suffix,
|
||||
"users": users,
|
||||
"controller": controller,
|
||||
"trainee_ids": trainee_ids,
|
||||
"trainee_names": trainee_names,
|
||||
}
|
||||
|
||||
|
||||
async def delete_distinct_users(database_url: str, fixtures: dict, session_ids: list[str]) -> None:
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.db.models import AuditLog, Session, Trainee, User
|
||||
|
||||
engine = create_async_engine(database_url)
|
||||
try:
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
async with factory() as db:
|
||||
ids = [UUID(value) for value in session_ids]
|
||||
logins = [item["login"] for item in fixtures["users"]]
|
||||
if fixtures["controller"]:
|
||||
logins.append(fixtures["controller"]["login"])
|
||||
if ids:
|
||||
await db.execute(delete(Session).where(Session.id.in_(ids)))
|
||||
await db.execute(delete(AuditLog).where(
|
||||
AuditLog.object_id.in_([str(value) for value in ids])
|
||||
))
|
||||
await db.execute(delete(AuditLog).where(AuditLog.actor.in_(logins)))
|
||||
await db.execute(delete(User).where(User.login.in_(logins)))
|
||||
await db.execute(delete(Trainee).where(
|
||||
Trainee.name.in_(fixtures["trainee_names"])
|
||||
))
|
||||
await db.commit()
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def session_node_distribution(database_url: str, session_ids: list[str]) -> dict[str, int]:
|
||||
"""Read the durable node owner for this run's sessions before cleanup."""
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.db.models import Session
|
||||
|
||||
engine = create_async_engine(database_url)
|
||||
try:
|
||||
async with async_sessionmaker(engine)() as db:
|
||||
rows = (await db.execute(
|
||||
select(Session.backend_node_id, func.count())
|
||||
.where(Session.id.in_([UUID(value) for value in session_ids]))
|
||||
.group_by(Session.backend_node_id)
|
||||
)).all()
|
||||
return {node or "<unassigned>": count for node, count in rows}
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def kill_cluster_service(project: str, service: str) -> None:
|
||||
"""SIGKILL one backend only in an explicitly named disposable Compose project."""
|
||||
if not project or not project.startswith("lct-"):
|
||||
raise ValueError("fault injection requires an explicit lct-* disposable Compose project")
|
||||
listing = subprocess.run(
|
||||
["docker", "ps", "--filter", f"label=com.docker.compose.project={project}",
|
||||
"--filter", f"label=com.docker.compose.service={service}", "--format", "{{.ID}}"],
|
||||
check=False, capture_output=True, text=True,
|
||||
)
|
||||
containers = [line.strip() for line in listing.stdout.splitlines() if line.strip()]
|
||||
if listing.returncode or len(containers) != 1:
|
||||
raise RuntimeError(
|
||||
f"expected exactly one running {service} in disposable project {project}; "
|
||||
f"found {len(containers)}"
|
||||
)
|
||||
completed = subprocess.run(
|
||||
["docker", "kill", containers[0]], check=False, capture_output=True, text=True,
|
||||
)
|
||||
if completed.returncode:
|
||||
raise RuntimeError(f"Docker SIGKILL failed: {completed.stderr.strip()}")
|
||||
|
||||
|
||||
async def wait_for_takeover(
|
||||
database_url: str, session_ids: list[str], *, failed_node: str, timeout: float,
|
||||
) -> tuple[dict[str, int], float]:
|
||||
started = time.perf_counter()
|
||||
deadline = started + timeout
|
||||
distribution: dict[str, int] = {}
|
||||
while time.perf_counter() < deadline:
|
||||
distribution = await session_node_distribution(database_url, session_ids)
|
||||
if sum(distribution.values()) == len(session_ids) and failed_node not in distribution:
|
||||
return distribution, (time.perf_counter() - started) * 1000
|
||||
await asyncio.sleep(0.5)
|
||||
return distribution, (time.perf_counter() - started) * 1000
|
||||
|
||||
|
||||
async def login_distinct_users(base: str, users: list[dict]) -> list[str]:
|
||||
semaphore = asyncio.Semaphore(8)
|
||||
|
||||
async def login(user: dict) -> str:
|
||||
async with semaphore:
|
||||
return await asyncio.to_thread(login_cookie, base, user["login"], user["password"])
|
||||
|
||||
return await asyncio.gather(*(login(user) for user in users))
|
||||
|
||||
|
||||
def local_url(value: str, schemes: set[str]) -> str:
|
||||
parsed = urllib.parse.urlsplit(value)
|
||||
|
|
@ -52,63 +233,256 @@ def login_cookie(base: str, login: str, password: str) -> str:
|
|||
return value
|
||||
|
||||
|
||||
async def control(ws_base: str, session_id: str, cookie: str, payload: dict) -> None:
|
||||
async def control(
|
||||
ws_base: str, session_id: str, cookie: str, payload: dict, *, ignore_https_errors: bool = False
|
||||
) -> None:
|
||||
header_arg = ("additional_headers" if "additional_headers" in inspect.signature(
|
||||
websockets.connect
|
||||
).parameters else "extra_headers")
|
||||
options = {header_arg: {"Cookie": cookie}, "open_timeout": 10, "ping_interval": None}
|
||||
if ws_base.startswith("wss://") and ignore_https_errors:
|
||||
options["ssl"] = ssl._create_unverified_context()
|
||||
async with websockets.connect(
|
||||
f"{ws_base}/ws/control/{session_id}",
|
||||
**{header_arg: {"Cookie": cookie}}, open_timeout=10, ping_interval=None,
|
||||
**options,
|
||||
) as socket:
|
||||
await socket.send(json.dumps(payload, ensure_ascii=False))
|
||||
await asyncio.sleep(0.03)
|
||||
|
||||
|
||||
async def probe_call_ws(
|
||||
ws_base: str, session_id: str, cookie: str, *, ignore_https_errors: bool,
|
||||
) -> dict:
|
||||
"""Probe a new call-channel handshake without competing for Chromium sockets."""
|
||||
header_arg = ("additional_headers" if "additional_headers" in inspect.signature(
|
||||
websockets.connect
|
||||
).parameters else "extra_headers")
|
||||
options = {
|
||||
header_arg: {"Cookie": cookie}, "open_timeout": 5, "ping_interval": None,
|
||||
}
|
||||
if ws_base.startswith("wss://") and ignore_https_errors:
|
||||
options["ssl"] = ssl._create_unverified_context()
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
async with websockets.connect(f"{ws_base}/ws/call/{session_id}", **options):
|
||||
return {"ok": True, "elapsed_ms": (time.perf_counter() - started) * 1000}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"ok": False,
|
||||
"elapsed_ms": (time.perf_counter() - started) * 1000,
|
||||
"reason": f"{type(exc).__name__}: {str(exc)[:240]}",
|
||||
}
|
||||
|
||||
|
||||
def track_page_websockets(page, records: list[dict]) -> None:
|
||||
"""Record only frame/close timestamps; never retain payloads or user data."""
|
||||
def on_websocket(socket) -> None:
|
||||
record = {
|
||||
"url": socket.url,
|
||||
"created_at": time.perf_counter(),
|
||||
"frames": [],
|
||||
"closed_at": None,
|
||||
}
|
||||
records.append(record)
|
||||
|
||||
def on_frame(frame, item=record) -> None:
|
||||
event_type = None
|
||||
error_code = None
|
||||
if isinstance(frame, str):
|
||||
try:
|
||||
value = json.loads(frame)
|
||||
if isinstance(value, dict):
|
||||
event_type = value.get("type")
|
||||
error_code = value.get("code")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
item["frames"].append({
|
||||
"at": time.perf_counter(), "type": event_type, "code": error_code,
|
||||
})
|
||||
|
||||
socket.on("framereceived", on_frame)
|
||||
socket.on("close", lambda _socket, item=record: item.update(
|
||||
closed_at=time.perf_counter()
|
||||
))
|
||||
|
||||
page.on("websocket", on_websocket)
|
||||
|
||||
|
||||
def browser_channels_recovered(
|
||||
records_by_page: list[list[dict]], expected_paths: list[str], after: float,
|
||||
) -> tuple[list[int], list[float]]:
|
||||
recovered: list[int] = []
|
||||
first_frame_after: list[float] = []
|
||||
for index, records in enumerate(records_by_page):
|
||||
candidates = []
|
||||
for record in records:
|
||||
if expected_paths[index] not in record["url"]:
|
||||
continue
|
||||
post_failure_frames = [
|
||||
frame["at"] for frame in record["frames"]
|
||||
if frame["at"] >= after and frame["type"] is not None
|
||||
and frame["type"] != "error" and frame["code"] is None
|
||||
]
|
||||
if (post_failure_frames
|
||||
and (record["closed_at"] is None
|
||||
or record["closed_at"] > post_failure_frames[-1])):
|
||||
candidates.append(post_failure_frames[0])
|
||||
if candidates:
|
||||
recovered.append(index)
|
||||
first_frame_after.append(min(candidates))
|
||||
return recovered, first_frame_after
|
||||
|
||||
|
||||
async def wait_for_browser_channels(
|
||||
records_by_page: list[list[dict]], expected_paths: list[str], after: float, timeout: float,
|
||||
) -> tuple[list[int], list[float]]:
|
||||
deadline = time.perf_counter() + max(0, timeout)
|
||||
while True:
|
||||
recovered, first_frames = browser_channels_recovered(
|
||||
records_by_page, expected_paths, after,
|
||||
)
|
||||
if len(recovered) == len(records_by_page) or time.perf_counter() >= deadline:
|
||||
return recovered, first_frames
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
frontend = local_url(args.frontend_url, {"http", "https"})
|
||||
backend = local_url(args.backend_url, {"http"})
|
||||
control_origin = local_url(args.control_url or args.backend_url, {"http", "https"})
|
||||
if bool(args.login) != bool(args.password):
|
||||
raise ValueError("--login и --password задаются вместе")
|
||||
if not args.distinct_users and not args.login:
|
||||
raise ValueError("обычный режим требует --login и --password")
|
||||
if args.sessions < 1 or args.clients < args.sessions or args.clients % args.sessions:
|
||||
raise ValueError("clients должно делиться на sessions и быть не меньше sessions")
|
||||
if args.clients > 100 or args.sessions > 20:
|
||||
raise ValueError("предохранитель: максимум 100 браузеров и 20 занятий")
|
||||
if args.distinct_users and (args.sessions != 20 or args.clients != 100):
|
||||
raise ValueError("--distinct-users проверяет ровно 20 занятий и 100 отдельных пользователей")
|
||||
if args.ramp_seconds < 0:
|
||||
raise ValueError("ramp-seconds не может быть отрицательным")
|
||||
if not 0 <= args.expect_backend_nodes <= args.sessions:
|
||||
raise ValueError("expect-backend-nodes должен быть от 0 до числа занятий")
|
||||
if args.kill_backend_b_after_pages:
|
||||
if not args.distinct_users or args.sessions != 20 or args.clients != 100:
|
||||
raise ValueError("cluster fault-injection requires --distinct-users with 20 sessions / 100 clients")
|
||||
if not args.database_url or not args.compose_project:
|
||||
raise ValueError("cluster fault-injection requires --database-url and --compose-project")
|
||||
if args.expect_backend_nodes < 2:
|
||||
raise ValueError("fault-injection requires --expect-backend-nodes 2")
|
||||
if control_origin != frontend or not frontend.startswith("https://"):
|
||||
raise ValueError("fault-injection requires TLS Nginx for both frontend and control-url")
|
||||
if args.failure_timeout <= 0:
|
||||
raise ValueError("failure-timeout must be positive")
|
||||
|
||||
cookie = login_cookie(backend, args.login, args.password)
|
||||
cookie_name, cookie_value = cookie.split("=", 1)
|
||||
ws_base = "ws" + backend.removeprefix("http")
|
||||
control_ws_base = control_origin.replace("https://", "wss://", 1).replace("http://", "ws://", 1)
|
||||
sessions = [str(uuid4()) for _ in range(args.sessions)]
|
||||
started: list[str] = []
|
||||
fixtures = None
|
||||
teacher_cookie = None
|
||||
try:
|
||||
await asyncio.gather(*(
|
||||
control(ws_base, session_id, cookie, {
|
||||
"type": "scenario.start", "scenario_id": args.scenario,
|
||||
"trainee": f"browser-load-{index:02d}", "mode": "training", "exercise": "dds",
|
||||
}) for index, session_id in enumerate(sessions)
|
||||
))
|
||||
station_cookies: list[str] = []
|
||||
observer_cookies: list[str] = []
|
||||
if args.distinct_users:
|
||||
if not args.database_url:
|
||||
raise ValueError("--distinct-users требует --database-url")
|
||||
fixtures = await create_distinct_users(
|
||||
args.database_url,
|
||||
args.sessions,
|
||||
args.clients - args.sessions,
|
||||
create_controller=not bool(args.login),
|
||||
)
|
||||
controller = fixtures["controller"]
|
||||
teacher_cookie = (
|
||||
login_cookie(backend, controller["login"], controller["password"])
|
||||
if controller
|
||||
else login_cookie(backend, args.login, args.password)
|
||||
)
|
||||
account_cookies = await login_distinct_users(backend, fixtures["users"])
|
||||
station_cookies = account_cookies[:args.sessions]
|
||||
observer_cookies = account_cookies[args.sessions:]
|
||||
else:
|
||||
if not args.login or not args.password:
|
||||
raise ValueError("обычный режим требует --login и --password")
|
||||
teacher_cookie = login_cookie(backend, args.login, args.password)
|
||||
|
||||
# Include partially started sessions in cleanup if one concurrent
|
||||
# control connection fails midway through the preparation batch.
|
||||
started.extend(sessions)
|
||||
await asyncio.gather(*(
|
||||
control(control_ws_base, session_id, teacher_cookie, {
|
||||
"type": "scenario.start", "scenario_id": args.scenario,
|
||||
"trainee": (
|
||||
fixtures["trainee_names"][index] if fixtures
|
||||
else f"browser-load-{index:02d}"
|
||||
),
|
||||
"trainee_id": fixtures["trainee_ids"][index] if fixtures else None,
|
||||
"mode": "training", "exercise": "dds",
|
||||
}, ignore_https_errors=args.ignore_https_errors)
|
||||
for index, session_id in enumerate(sessions)
|
||||
))
|
||||
|
||||
timings: list[float] = []
|
||||
dds_timings: list[float] = []
|
||||
observer_timings: list[float] = []
|
||||
browser_navigation: list[dict] = []
|
||||
page_readiness_profiles: list[dict] = []
|
||||
errors: list[str] = []
|
||||
pages = []
|
||||
contexts = []
|
||||
per_session = args.clients // args.sessions
|
||||
observer_index = 0
|
||||
failure_injection = None
|
||||
pre_failure_distribution: dict[str, int] = {}
|
||||
websocket_records: list[list[dict]] = [[] for _ in range(args.clients)]
|
||||
expected_ws_paths: list[str] = []
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
for index in range(args.clients):
|
||||
session_index = index // per_session
|
||||
role_index = index % per_session
|
||||
if fixtures:
|
||||
account_cookie = (
|
||||
station_cookies[session_index]
|
||||
if role_index == 0
|
||||
else observer_cookies[observer_index]
|
||||
)
|
||||
if role_index != 0:
|
||||
observer_index += 1
|
||||
else:
|
||||
account_cookie = teacher_cookie
|
||||
cookie_name, cookie_value = account_cookie.split("=", 1)
|
||||
context = await browser.new_context(
|
||||
viewport={"width": 1280, "height": 720},
|
||||
ignore_https_errors=args.ignore_https_errors,
|
||||
)
|
||||
await context.add_init_script("""
|
||||
window.__lctLongTasks = [];
|
||||
if (typeof PerformanceObserver !== "undefined") {
|
||||
try {
|
||||
new PerformanceObserver((list) => {
|
||||
for (const item of list.getEntries()) {
|
||||
window.__lctLongTasks.push({ start: item.startTime, duration: item.duration });
|
||||
}
|
||||
}).observe({ type: "longtask", buffered: true });
|
||||
} catch (_) {}
|
||||
}
|
||||
""")
|
||||
await context.add_cookies([{
|
||||
"name": cookie_name, "value": cookie_value, "url": frontend,
|
||||
"httpOnly": True, "sameSite": "Lax",
|
||||
}])
|
||||
page = await context.new_page()
|
||||
page.on("pageerror", lambda error, i=index: errors.append(f"page {i}: {error}"))
|
||||
session_index = index // per_session
|
||||
role_index = index % per_session
|
||||
session_id = sessions[session_index]
|
||||
expected_ws_paths.append(
|
||||
f"/ws/station/{session_id}" if role_index == 0
|
||||
else f"/ws/observe/{session_id}"
|
||||
)
|
||||
track_page_websockets(page, websocket_records[index])
|
||||
contexts.append(context)
|
||||
pages.append(page)
|
||||
|
||||
|
|
@ -131,6 +505,7 @@ async def run(args: argparse.Namespace) -> int:
|
|||
)
|
||||
if response is None or not response.ok:
|
||||
raise RuntimeError(f"HTTP {response.status if response else 'нет ответа'}")
|
||||
document_ready_ms = (time.perf_counter() - started_at) * 1000
|
||||
await pages[index].locator(selector).wait_for(timeout=args.timeout * 1000)
|
||||
if role_index != 0:
|
||||
await pages[index].wait_for_function(
|
||||
|
|
@ -140,6 +515,68 @@ async def run(args: argparse.Namespace) -> int:
|
|||
ready_ms = (time.perf_counter() - started_at) * 1000
|
||||
timings.append(ready_ms)
|
||||
(dds_timings if role_index == 0 else observer_timings).append(ready_ms)
|
||||
profile = await pages[index].evaluate("""() => {
|
||||
const resources = performance.getEntriesByType("resource")
|
||||
.map((item) => {
|
||||
const path = new URL(item.name).pathname.replace(
|
||||
/[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}/gi, ":id"
|
||||
);
|
||||
return {
|
||||
path,
|
||||
initiator: item.initiatorType,
|
||||
duration_ms: item.duration,
|
||||
transfer_bytes: item.transferSize
|
||||
};
|
||||
})
|
||||
.filter((item) => item.path.startsWith("/assets/")
|
||||
|| item.path.startsWith("/api/"))
|
||||
.sort((left, right) => right.duration_ms - left.duration_ms)
|
||||
.slice(0, 30);
|
||||
const tasks = window.__lctLongTasks ?? [];
|
||||
return {
|
||||
long_tasks: tasks.map((item) => ({
|
||||
start_ms: item.start, duration_ms: item.duration
|
||||
})),
|
||||
resources
|
||||
};
|
||||
}""")
|
||||
profile["websockets"] = []
|
||||
for socket in websocket_records[index]:
|
||||
socket_path = urllib.parse.urlsplit(socket["url"]).path
|
||||
if socket_path.startswith("/ws/station/"):
|
||||
socket_path = "/ws/station/:id"
|
||||
elif socket_path.startswith("/ws/observe/"):
|
||||
socket_path = "/ws/observe/:id"
|
||||
elif socket_path.startswith("/ws/control/"):
|
||||
socket_path = "/ws/control/:id"
|
||||
frame_types = [frame["type"] for frame in socket["frames"]]
|
||||
first_state = next((
|
||||
frame for frame in socket["frames"]
|
||||
if frame["type"] in {
|
||||
"station.state", "observe.state", "session.snapshot",
|
||||
"card.received",
|
||||
}
|
||||
), None)
|
||||
profile["websockets"].append({
|
||||
"path": socket_path,
|
||||
"created_after_navigation_ms": round(
|
||||
(socket["created_at"] - started_at) * 1000, 2
|
||||
),
|
||||
"first_frame_after_navigation_ms": round(
|
||||
(socket["frames"][0]["at"] - started_at) * 1000, 2
|
||||
) if socket["frames"] else None,
|
||||
"first_state_after_navigation_ms": round(
|
||||
(first_state["at"] - started_at) * 1000, 2
|
||||
) if first_state else None,
|
||||
"first_frame_type": frame_types[0] if frame_types else None,
|
||||
})
|
||||
page_readiness_profiles.append({
|
||||
"page_index": index,
|
||||
"role": "dds" if role_index == 0 else "wall",
|
||||
"navigation_to_domcontentloaded_ms": round(document_ready_ms, 2),
|
||||
"domcontentloaded_to_ready_ms": round(ready_ms - document_ready_ms, 2),
|
||||
**profile,
|
||||
})
|
||||
navigation = await pages[index].evaluate("""() => {
|
||||
const entry = performance.getEntriesByType('navigation')[0];
|
||||
if (!entry) return null;
|
||||
|
|
@ -157,14 +594,182 @@ async def run(args: argparse.Namespace) -> int:
|
|||
errors.append(f"page {index} {path}: {type(exc).__name__}: {exc}")
|
||||
|
||||
await asyncio.gather(*(open_page(index) for index in range(args.clients)))
|
||||
|
||||
session_api_ok = 0
|
||||
call_ws_ok = 0
|
||||
route_errors: list[str] = []
|
||||
if args.kill_backend_b_after_pages:
|
||||
if len(timings) != args.clients or errors:
|
||||
raise RuntimeError("all 100 pages must be ready before failure injection")
|
||||
initial_ws_connected = sum(
|
||||
any(
|
||||
expected_ws_paths[index] in record["url"]
|
||||
and any(
|
||||
frame["type"] is not None
|
||||
and frame["type"] != "error" and frame["code"] is None
|
||||
for frame in record["frames"]
|
||||
)
|
||||
for record in records
|
||||
)
|
||||
for index, records in enumerate(websocket_records)
|
||||
)
|
||||
if initial_ws_connected != args.clients:
|
||||
raise RuntimeError(
|
||||
f"only {initial_ws_connected}/{args.clients} pages received their initial WS frame"
|
||||
)
|
||||
pre_failure_distribution = await session_node_distribution(
|
||||
args.database_url, sessions,
|
||||
)
|
||||
if set(pre_failure_distribution) != {"backend-a", "backend-b"}:
|
||||
raise RuntimeError(
|
||||
"pre-failure sessions must be owned by both backend-a and backend-b"
|
||||
)
|
||||
failure_started = time.perf_counter()
|
||||
kill_cluster_service(args.compose_project, "backend-b")
|
||||
post_failure_distribution, takeover_ms = await wait_for_takeover(
|
||||
args.database_url, sessions, failed_node="backend-b",
|
||||
timeout=args.failure_timeout,
|
||||
)
|
||||
recovery_budget = max(
|
||||
0.0,
|
||||
min(30.0, args.failure_timeout) - (time.perf_counter() - failure_started),
|
||||
)
|
||||
recovered_pages, first_frames = await wait_for_browser_channels(
|
||||
websocket_records, expected_ws_paths, failure_started, recovery_budget,
|
||||
)
|
||||
failure_injection = {
|
||||
"action": "SIGKILL backend-b after all pages became ready",
|
||||
"compose_project": args.compose_project,
|
||||
"owner_distribution_before": pre_failure_distribution,
|
||||
"owner_distribution_after": post_failure_distribution,
|
||||
"takeover_ms": round(takeover_ms, 2),
|
||||
"concurrent_pages_kept_open_during_takeover": args.clients,
|
||||
"takeover_passed": (
|
||||
sum(post_failure_distribution.values()) == args.sessions
|
||||
and post_failure_distribution.get("backend-a") == args.sessions
|
||||
and "backend-b" not in post_failure_distribution
|
||||
),
|
||||
"browser_ws_channel_pages_recovered": len(recovered_pages),
|
||||
"browser_ws_channel_pages_expected": args.clients,
|
||||
"browser_ws_initial_pages_connected": initial_ws_connected,
|
||||
"browser_ws_unrecovered_pages": [
|
||||
{
|
||||
"page_index": index,
|
||||
"channel": expected_ws_paths[index],
|
||||
"session_id": sessions[index // per_session],
|
||||
}
|
||||
for index in range(args.clients) if index not in recovered_pages
|
||||
],
|
||||
"browser_ws_channels_recovered_within_30s": (
|
||||
len(recovered_pages) == args.clients
|
||||
and bool(first_frames)
|
||||
and max(first_frames) - failure_started <= 30.0
|
||||
),
|
||||
"browser_ws_recovery_ms": (
|
||||
round((max(first_frames) - failure_started) * 1000, 2)
|
||||
if len(recovered_pages) == args.clients else None
|
||||
),
|
||||
}
|
||||
failure_injection["elapsed_since_kill_ms"] = round(
|
||||
(time.perf_counter() - failure_started) * 1000, 2
|
||||
)
|
||||
mounted = await asyncio.gather(*(
|
||||
page.evaluate("Boolean(document.querySelector('#root')?.firstElementChild)")
|
||||
for page in pages
|
||||
))
|
||||
failure_injection["react_pages_still_mounted"] = sum(bool(item) for item in mounted)
|
||||
|
||||
async def probe_session_api(session_index: int) -> dict:
|
||||
page = pages[session_index * per_session]
|
||||
return await page.evaluate("""async (sessionId) => {
|
||||
const started = performance.now();
|
||||
const response = await fetch(`/api/sessions/${sessionId}`, {
|
||||
credentials: "same-origin"
|
||||
});
|
||||
return {
|
||||
ok: response.status === 200,
|
||||
status: response.status,
|
||||
elapsed_ms: performance.now() - started
|
||||
};
|
||||
}""", sessions[session_index])
|
||||
|
||||
api_probes = await asyncio.gather(*(
|
||||
probe_session_api(session_index) for session_index in range(args.sessions)
|
||||
))
|
||||
ws_probe_cookies = (
|
||||
station_cookies if fixtures else [teacher_cookie] * args.sessions
|
||||
)
|
||||
ws_probes = await asyncio.gather(*(
|
||||
probe_call_ws(
|
||||
control_ws_base, sessions[index], ws_probe_cookies[index],
|
||||
ignore_https_errors=args.ignore_https_errors,
|
||||
)
|
||||
for index in range(args.sessions)
|
||||
))
|
||||
if failure_injection:
|
||||
failure_injection["route_probe_ms"] = {
|
||||
"session_api_max": round(max(item["elapsed_ms"] for item in api_probes), 2),
|
||||
"call_ws_max": round(max(item["elapsed_ms"] for item in ws_probes), 2),
|
||||
"call_ws_probe_client": "Python websockets; Chromium pages remain open",
|
||||
}
|
||||
for session_index, (api_probe, ws_probe) in enumerate(zip(api_probes, ws_probes)):
|
||||
if api_probe["ok"]:
|
||||
session_api_ok += 1
|
||||
else:
|
||||
route_errors.append(
|
||||
f"session {sessions[session_index]}: GET session returned {api_probe['status']}"
|
||||
)
|
||||
if ws_probe["ok"]:
|
||||
call_ws_ok += 1
|
||||
else:
|
||||
route_errors.append(
|
||||
f"session {sessions[session_index]}: /ws/call route failed: {ws_probe}"
|
||||
)
|
||||
|
||||
await asyncio.sleep(args.hold_seconds)
|
||||
await asyncio.gather(*(context.close() for context in contexts))
|
||||
await browser.close()
|
||||
|
||||
node_distribution = (
|
||||
await session_node_distribution(args.database_url, sessions)
|
||||
if args.database_url else {}
|
||||
)
|
||||
nodes_for_expectation = pre_failure_distribution if failure_injection else node_distribution
|
||||
expected_nodes_ready = (
|
||||
len(nodes_for_expectation) >= args.expect_backend_nodes
|
||||
if args.expect_backend_nodes else True
|
||||
)
|
||||
if failure_injection:
|
||||
failure_injection["session_rest_routes_recovered"] = session_api_ok == args.sessions
|
||||
failure_injection["call_ws_routes_recovered"] = call_ws_ok == args.sessions
|
||||
failure_injection["route_errors_after_takeover"] = list(route_errors)
|
||||
failure_injection["all_probes_recovered"] = (
|
||||
session_api_ok == args.sessions and call_ws_ok == args.sessions
|
||||
and not route_errors
|
||||
)
|
||||
failure_injection["pass_20_session_takeover"] = (
|
||||
failure_injection["takeover_passed"]
|
||||
and failure_injection["all_probes_recovered"]
|
||||
and failure_injection["browser_ws_channels_recovered_within_30s"]
|
||||
and failure_injection["react_pages_still_mounted"] == args.clients
|
||||
)
|
||||
result = {
|
||||
"frontend": frontend, "backend": backend,
|
||||
"frontend": frontend, "backend": backend, "control": control_origin,
|
||||
"sessions_started": len(sessions), "browser_contexts": args.clients,
|
||||
"dds_pages": args.sessions, "observer_pages": args.clients - args.sessions,
|
||||
"distinct_authenticated_users": (
|
||||
args.clients if fixtures else 1
|
||||
),
|
||||
"distinct_users_mode": bool(fixtures),
|
||||
"backend_node_distribution": node_distribution,
|
||||
"backend_node_count": len(node_distribution),
|
||||
"backend_node_distribution_before_failure": (
|
||||
pre_failure_distribution if failure_injection else None
|
||||
),
|
||||
"failure_injection": failure_injection,
|
||||
"session_rest_routes_ok": session_api_ok,
|
||||
"call_ws_routes_ok": call_ws_ok,
|
||||
"route_errors": route_errors,
|
||||
"pages_ready": len(timings), "hold_seconds": args.hold_seconds,
|
||||
"ramp_seconds": args.ramp_seconds,
|
||||
"ready_ms": timing_summary(timings),
|
||||
|
|
@ -172,6 +777,14 @@ async def run(args: argparse.Namespace) -> int:
|
|||
"dds": timing_summary(dds_timings),
|
||||
"wall": timing_summary(observer_timings),
|
||||
},
|
||||
"page_readiness_profiles": page_readiness_profiles,
|
||||
"pass_wall_report_deferred": not any(
|
||||
path == "/api/sessions/:id/report"
|
||||
or path.startswith("/assets/Debrief-")
|
||||
or path.startswith("/assets/Radar-")
|
||||
for profile in page_readiness_profiles if profile.get("role") == "wall"
|
||||
for path in (resource.get("path", "") for resource in profile.get("resources", []))
|
||||
),
|
||||
"browser_navigation_ms": {
|
||||
key: timing_summary([float(item[key]) for item in browser_navigation])
|
||||
for key in ("responseStart", "responseEnd", "domInteractive", "domContentLoaded")
|
||||
|
|
@ -181,13 +794,30 @@ async def run(args: argparse.Namespace) -> int:
|
|||
]),
|
||||
"errors": errors[:50],
|
||||
"pass_browser_ui_2s": (
|
||||
len(timings) == args.clients and not errors and percentile(timings, 95) <= 2000
|
||||
len(timings) == args.clients
|
||||
and not errors
|
||||
and not route_errors
|
||||
and session_api_ok == args.sessions
|
||||
and call_ws_ok == args.sessions
|
||||
and percentile(timings, 95) <= 2000
|
||||
and len(dds_timings) == args.sessions
|
||||
and len(observer_timings) == args.clients - args.sessions
|
||||
and percentile(dds_timings, 95) <= 2000
|
||||
and percentile(observer_timings, 95) <= 2000
|
||||
and expected_nodes_ready
|
||||
),
|
||||
"pass_failover_under_20x100": bool(
|
||||
failure_injection and failure_injection["pass_20_session_takeover"]
|
||||
) if args.kill_backend_b_after_pages else None,
|
||||
"scope": (
|
||||
f"{args.clients} isolated Chromium contexts render real React DDS/wall pages "
|
||||
f"for {args.sessions} sessions with a {args.ramp_seconds:g}s connection ramp; "
|
||||
"all contexts remain active concurrently; same instructor account, local host, no VoIP, "
|
||||
"all contexts remain active concurrently; "
|
||||
+ ("unique trainee/admin accounts" if fixtures else "same instructor account")
|
||||
+ ", local host, no VoIP, "
|
||||
"not target 6-core/32-GB server"
|
||||
+ (f"; at least {args.expect_backend_nodes} backend nodes required"
|
||||
if args.expect_backend_nodes else "")
|
||||
),
|
||||
}
|
||||
rendered = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
|
|
@ -195,20 +825,32 @@ async def run(args: argparse.Namespace) -> int:
|
|||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as stream:
|
||||
stream.write(rendered + "\n")
|
||||
return 0 if result["pass_browser_ui_2s"] else 1
|
||||
passed = result["pass_browser_ui_2s"] and (
|
||||
result["pass_wall_report_deferred"]
|
||||
and result["pass_failover_under_20x100"] is not False
|
||||
)
|
||||
return 0 if passed else 1
|
||||
finally:
|
||||
await asyncio.gather(*(
|
||||
control(ws_base, session_id, cookie, {"type": "session.stop"})
|
||||
for session_id in started
|
||||
), return_exceptions=True)
|
||||
try:
|
||||
if teacher_cookie and started:
|
||||
await asyncio.gather(*(
|
||||
control(control_ws_base, session_id, teacher_cookie, {"type": "session.stop"},
|
||||
ignore_https_errors=args.ignore_https_errors)
|
||||
for session_id in started
|
||||
), return_exceptions=True)
|
||||
finally:
|
||||
if fixtures:
|
||||
await delete_distinct_users(args.database_url, fixtures, started)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--frontend-url", default="http://127.0.0.1:5173")
|
||||
parser.add_argument("--backend-url", default="http://127.0.0.1:8000")
|
||||
parser.add_argument("--login", required=True)
|
||||
parser.add_argument("--password", required=True)
|
||||
parser.add_argument("--control-url",
|
||||
help="URL для управляющего WS; задайте frontend URL для проверки proxy-affinity")
|
||||
parser.add_argument("--login", help="преподаватель; в --distinct-users создаётся временный")
|
||||
parser.add_argument("--password", help="пароль преподавателя, если задан --login")
|
||||
parser.add_argument("--sessions", type=int, default=20)
|
||||
parser.add_argument("--clients", type=int, default=100)
|
||||
parser.add_argument("--scenario", default="fire-apartment-l2")
|
||||
|
|
@ -216,6 +858,18 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--hold-seconds", type=float, default=2)
|
||||
parser.add_argument("--ramp-seconds", type=float, default=0,
|
||||
help="равномерно распределить начало загрузок; все страницы остаются активны")
|
||||
parser.add_argument("--distinct-users", action="store_true",
|
||||
help="создать/удалить временные учётки и проверить разные логины")
|
||||
parser.add_argument("--database-url",
|
||||
help="PostgreSQL URL, обязателен для --distinct-users; только тестовая БД")
|
||||
parser.add_argument("--expect-backend-nodes", type=int, default=0,
|
||||
help="проверить, что занятия распределены минимум по N backend-узлам")
|
||||
parser.add_argument("--kill-backend-b-after-pages", action="store_true",
|
||||
help="SIGKILL backend-b after all 100 pages load; isolated cluster only")
|
||||
parser.add_argument("--compose-project",
|
||||
help="required unique lct-* project for --kill-backend-b-after-pages")
|
||||
parser.add_argument("--failure-timeout", type=float, default=45,
|
||||
help="maximum seconds to wait for leases/checkpoints to transfer")
|
||||
parser.add_argument("--output", help="сохранить JSON-отчёт")
|
||||
parser.add_argument("--ignore-https-errors", action="store_true",
|
||||
help="разрешить bootstrap self-signed сертификат только в локальном тесте")
|
||||
|
|
|
|||
Loading…
Reference in a new issue