pub/lct-hack
Watch
1
0
Fork
You've already forked lct-hack
 0
Code Releases Packages 5 Activity

Complete training workflow and acceptance hardening

Browse source
This commit is contained in:
andreysk0304 2026-09-26 18:12:27 +03:00 • committed by gglamer
commit 7237265833
243 changed files with 17014 additions and 1500 deletions
Download patch file Download diff file Expand all files Collapse all files

199
scripts/test_recovery.py
View file

@ -12,6 +12,7 @@ from uuid import uuid4
from playwright.async_api import async_playwright
from sqlalchemy import delete
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from load_browser import control, local_url, login_cookie
@ -67,23 +68,27 @@ async def cleanup_temp_accounts(
logins: list[str],
trainee_name: str,
session_id,
*,
remove_accounts: bool,
) -> None:
engine = create_async_engine(database_url)
try:
factory = async_sessionmaker(engine, expire_on_commit=False)
async with factory() as db:
await db.execute(delete(Session).where(Session.id == session_id))
await db.execute(delete(User).where(User.login.in_(logins)))
await db.execute(delete(Trainee).where(Trainee.name == trainee_name))
await db.execute(delete(AuditLog).where(AuditLog.actor.in_(logins)))
await db.execute(delete(AuditLog).where(AuditLog.object_id == str(session_id)))
if remove_accounts:
await db.execute(delete(AuditLog).where(AuditLog.actor.in_(logins)))
await db.execute(delete(User).where(User.login.in_(logins)))
await db.execute(delete(Trainee).where(Trainee.name == trainee_name))
await db.commit()
finally:
await engine.dispose()
async def compose_service(action: str, service: str) -> None:
async def compose_service(project: str, action: str, service: str) -> None:
process = await asyncio.create_subprocess_exec(
"docker", "compose", action, service,
"docker", "compose", "--project-name", project, action, service,
cwd=ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
@ -94,25 +99,13 @@ async def compose_service(action: str, service: str) -> None:
raise RuntimeError(f"docker compose {action} {service}: {detail}")
async def wait_service_healthy(service: str, timeout: float = 45) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
process = await asyncio.create_subprocess_exec(
"docker", "compose", "ps", service,
cwd=ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, _ = await process.communicate()
if process.returncode == 0 and b"(healthy)" in stdout:
return
await asyncio.sleep(0.5)
raise TimeoutError(f"контейнер {service} не стал healthy за {timeout} с")
async def run(args: argparse.Namespace) -> int:
frontend = local_url(args.frontend_url, {"https"})
backend = local_url(args.backend_url, {"http"})
if not args.compose_project.strip():
raise ValueError("--compose-project обязателен: recovery test останавливает Nginx")
if not 0 <= args.outage_seconds <= 30:
raise ValueError("--outage-seconds должен быть от 0 до 30")
ephemeral = not args.login and not args.password
if bool(args.login) != bool(args.password):
raise ValueError("login и password задаются вместе либо не задаются")
@ -120,7 +113,7 @@ async def run(args: argparse.Namespace) -> int:
instructor_password = args.password
trainee_login = args.trainee_login
trainee_password = args.trainee_password
trainee_name = "recovery-check"
trainee_name = ""
trainee_id = None
session_uuid = uuid4()
if ephemeral:
@ -138,8 +131,26 @@ async def run(args: argparse.Namespace) -> int:
trainee_password,
trainee_name,
)
elif not trainee_login or not trainee_password:
raise ValueError("для существующих аккаунтов нужны trainee-login и trainee-password")
else:
if not trainee_login or not trainee_password:
raise ValueError("для существующих аккаунтов нужны trainee-login и trainee-password")
engine = create_async_engine(args.database_url)
try:
async with async_sessionmaker(engine, expire_on_commit=False)() as db:
row = (await db.execute(
select(Trainee.id, Trainee.name)
.join(User, User.trainee_id == Trainee.id)
.where(
User.login == trainee_login,
User.role == "trainee",
User.blocked.is_(False),
)
)).one_or_none()
if row is None:
raise ValueError("учётка курсанта не найдена или не привязана к профилю")
trainee_id, trainee_name = row
finally:
await engine.dispose()
assert instructor_login and instructor_password and trainee_login and trainee_password
instructor_cookie = login_cookie(backend, instructor_login, instructor_password)
trainee_cookie = login_cookie(backend, trainee_login, trainee_password)
@ -148,7 +159,6 @@ async def run(args: argparse.Namespace) -> int:
session_id = str(session_uuid)
started = False
frontend_stopped = False
backend_paused = False
try:
await control(ws_base, session_id, instructor_cookie, {
"type": "scenario.start",
@ -176,6 +186,41 @@ async def run(args: argparse.Namespace) -> int:
page = await context.new_page()
errors: list[str] = []
page.on("pageerror", lambda error: errors.append(str(error)))
await page.add_init_script("""(() => {
window.__lctDropOperatorKioAck = false;
window.__lctOperatorAckDropped = false;
let owner = WebSocket.prototype;
let descriptor;
while (owner && !descriptor) {
descriptor = Object.getOwnPropertyDescriptor(owner, 'onmessage');
if (!descriptor) owner = Object.getPrototypeOf(owner);
}
if (!owner || !descriptor?.set || !descriptor?.get) return;
Object.defineProperty(owner, 'onmessage', {
configurable: descriptor.configurable,
enumerable: descriptor.enumerable,
get() { return descriptor.get.call(this); },
set(handler) {
const socket = this;
const wrapped = (event) => {
if (window.__lctDropOperatorKioAck && typeof event.data === 'string') {
try {
const message = JSON.parse(event.data);
if (message.type === 'kio.patch' && message.source === 'operator') {
window.__lctDropOperatorKioAck = false;
window.__lctOperatorAckDropped = true;
socket.close(4000, 'test: discard checkpoint acknowledgement');
return;
}
} catch { /* non-JSON frames are passed to the application */ }
}
if (handler) handler.call(socket, event);
};
descriptor.set.call(socket, handler ? wrapped : null);
},
});
window.__lctAckInterceptorInstalled = Boolean(descriptor?.set && descriptor?.get);
})()""")
await page.goto(
f"{frontend}/trainee?session={session_id}",
wait_until="domcontentloaded",
@ -185,6 +230,8 @@ async def run(args: argparse.Namespace) -> int:
await address.wait_for(
timeout=args.timeout * 1000,
)
if not await page.evaluate("window.__lctAckInterceptorInstalled"):
raise RuntimeError("browser could not install the WebSocket acknowledgement test hook")
await page.locator(".arm-topbar-state .state-ok").wait_for(
timeout=args.timeout * 1000,
)
@ -198,36 +245,70 @@ async def run(args: argparse.Namespace) -> int:
buffered_value = "улица Буферная, дом 30"
started_at = time.perf_counter()
# Держим существующее WSS-соединение открытым, но не даём backend
# подтвердить правку. Это воспроизводит пакет, зависший ровно в
# момент отказа прокси, без искусственного доступа к JS-сокету.
await compose_service("pause", "backend")
backend_paused = True
await address.fill(buffered_value)
await page.locator(".kio-arm-row-address.kio-arm-pending").wait_for(
timeout=args.timeout * 1000,
)
await compose_service("stop", "frontend")
# Останавливаем proxy и вводим значение только после того, как
# browser отметил канал отключённым: это проверяет offline outbox.
await compose_service(args.compose_project, "stop", "frontend")
frontend_stopped = True
outage_started_at = time.perf_counter()
await page.wait_for_function(
"window.__lctRecoveryStates.some(value => !value.includes('АРМ подключён'))",
timeout=args.timeout * 1000,
)
await compose_service("unpause", "backend")
backend_paused = False
await wait_service_healthy("backend")
await compose_service("start", "frontend")
# The operator edits only after the channel is known to be down.
# This exercises the disconnected outbox path instead of relying
# on a frame which might already have reached the server.
await address.fill(buffered_value)
await page.locator(".kio-arm-row-address.kio-arm-pending").wait_for(
timeout=args.timeout * 1000,
)
outage_remaining = args.outage_seconds - (time.perf_counter() - outage_started_at)
if outage_remaining > 0:
await asyncio.sleep(outage_remaining)
restore_started_at = time.perf_counter()
proxy_unavailable_seconds = restore_started_at - outage_started_at
await compose_service(args.compose_project, "start", "frontend")
frontend_stopped = False
await page.wait_for_function(
"window.__lctRecoveryStates.at(-1).includes('АРМ подключён')",
timeout=args.timeout * 1000,
)
recovery_seconds = time.perf_counter() - restore_started_at
await page.wait_for_function(
"!document.querySelector('.kio-arm-row-address')?.classList.contains('kio-arm-pending')",
timeout=args.timeout * 1000,
)
buffered_value_after = await page.locator(".kio-arm-row-address input").input_value()
# Now send while the WSS is open, allow PostgreSQL checkpoint/echo
# to complete, then deliberately discard that one echo and close
# the browser socket. This targets the ambiguous commit/ack window:
# reconnect must recover the stored value without duplicating a
# non-replace-style action or leaving the field pending.
committed_value = "улица Подтверждённая, дом 31"
await page.evaluate("window.__lctDropOperatorKioAck = true")
await address.fill(committed_value)
await page.wait_for_function(
"window.__lctOperatorAckDropped === true",
timeout=args.timeout * 1000,
)
await page.wait_for_function(
"window.__lctRecoveryStates.some(value => !value.includes('АРМ подключён'))",
timeout=args.timeout * 1000,
)
await page.wait_for_function(
"window.__lctRecoveryStates.at(-1).includes('АРМ подключён')",
timeout=args.timeout * 1000,
)
await page.wait_for_function(
"!document.querySelector('.kio-arm-row-address')?.classList.contains('kio-arm-pending')",
timeout=args.timeout * 1000,
)
buffered_value_after = await page.locator(".kio-arm-row-address input").input_value()
recovery_seconds = time.perf_counter() - started_at
committed_value_after = await page.locator(".kio-arm-row-address input").input_value()
ack_deliberately_dropped = await page.evaluate("window.__lctOperatorAckDropped")
pending_cleared = not await page.locator(".kio-arm-row-address").evaluate(
"element => element.classList.contains('kio-arm-pending')"
)
total_seconds = time.perf_counter() - started_at
states = await page.evaluate("window.__lctRecoveryStates")
await context.close()
await browser.close()
@ -237,19 +318,37 @@ async def run(args: argparse.Namespace) -> int:
"session_id": session_id,
"component_restarted": "frontend (Nginx TLS/WSS proxy) container stop/start",
"observed_states": states,
"recovery_seconds": recovery_seconds,
"requested_outage_seconds": args.outage_seconds,
"proxy_unavailable_seconds": proxy_unavailable_seconds,
"recovery_after_restore_seconds": recovery_seconds,
"total_seconds": total_seconds,
"browser_errors": errors,
"buffered_patch": {
"field": "address",
"value": buffered_value_after,
"confirmed_by_server": buffered_value_after == buffered_value,
},
"lost_ack_recovery": {
"ack_deliberately_dropped": ack_deliberately_dropped,
"value_after_reconnect": committed_value_after,
"confirmed_by_recovered_server_state": committed_value_after == committed_value,
"pending_cleared": pending_cleared,
},
"pass_recovery_30s": (
recovery_seconds <= 30 and not errors and buffered_value_after == buffered_value
args.outage_seconds <= 30
and proxy_unavailable_seconds <= 30.5
and recovery_seconds <= 30
and not errors
and buffered_value_after == buffered_value
and committed_value_after == committed_value
and ack_deliberately_dropped
and pending_cleared
),
"scope": (
"one live card exercise; backend acknowledgement is stalled after address edit; "
"Nginx is stopped; after recovery reconnect repeats only the idempotent KIO patch"
"one live card exercise; Nginx is unavailable while the trainee enters an address; "
"the disconnected KIO patch is queued and confirmed after WSS reconnect; a second "
"KIO patch is committed while connected, its checkpoint echo is deliberately "
"discarded, and reconnect recovers that state; backend remains running"
),
}
rendered = json.dumps(result, ensure_ascii=False, indent=2)
@ -259,11 +358,8 @@ async def run(args: argparse.Namespace) -> int:
stream.write(rendered + "\n")
return 0 if result["pass_recovery_30s"] else 1
finally:
if backend_paused:
await compose_service("unpause", "backend")
await wait_service_healthy("backend")
if frontend_stopped:
await compose_service("start", "frontend")
await compose_service(args.compose_project, "start", "frontend")
if started:
await control(ws_base, session_id, instructor_cookie, {"type": "session.stop"})
if ephemeral:
@ -272,6 +368,7 @@ async def run(args: argparse.Namespace) -> int:
[instructor_login, trainee_login],
trainee_name,
session_uuid,
remove_accounts=ephemeral,
)
@ -279,6 +376,8 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--frontend-url", default="https://127.0.0.1:5443")
parser.add_argument("--backend-url", default="http://127.0.0.1:8000")
parser.add_argument("--compose-project", required=True,
help="точное имя изолированного Compose project; тест останавливает его frontend")
parser.add_argument("--login")
parser.add_argument("--password")
parser.add_argument("--trainee-login")
@ -289,6 +388,8 @@ if __name__ == "__main__":
)
parser.add_argument("--scenario", default="fire-apartment-l2")
parser.add_argument("--timeout", type=float, default=30)
parser.add_argument("--outage-seconds", type=float, default=0,
help="держать frontend недоступным столько секунд (0–30), вводить карточку после разрыва")
parser.add_argument("--ignore-https-errors", action="store_true")
parser.add_argument("--output")
try: