fix: повторять сохранение присутствия после сбоя
This commit is contained in:
parent
57c3ff5a5c
commit
9dd7ac425b
2 changed files with 48 additions and 0 deletions
|
|
@ -56,6 +56,7 @@ class SessionHub:
|
||||||
self._stations: dict[UUID, set[asyncio.Queue]] = {}
|
self._stations: dict[UUID, set[asyncio.Queue]] = {}
|
||||||
self._trainee_calls: dict[UUID, set[asyncio.Queue]] = {}
|
self._trainee_calls: dict[UUID, set[asyncio.Queue]] = {}
|
||||||
self._trainee_stations: dict[UUID, set[asyncio.Queue]] = {}
|
self._trainee_stations: dict[UUID, set[asyncio.Queue]] = {}
|
||||||
|
self._pending_adoptions: dict[UUID, SessionState] = {}
|
||||||
self._tickers: dict[UUID, asyncio.Task] = {}
|
self._tickers: dict[UUID, asyncio.Task] = {}
|
||||||
# Задачи, порождённые внутри операции, наследуют контекст; после
|
# Задачи, порождённые внутри операции, наследуют контекст; после
|
||||||
# закрытия операции их события идут напрямую (`_Operation.open`).
|
# закрытия операции их события идут напрямую (`_Operation.open`).
|
||||||
|
|
@ -119,6 +120,7 @@ class SessionHub:
|
||||||
|
|
||||||
def drop(self, session_id: UUID) -> None:
|
def drop(self, session_id: UUID) -> None:
|
||||||
self._sessions.pop(session_id, None)
|
self._sessions.pop(session_id, None)
|
||||||
|
self._pending_adoptions.pop(session_id, None)
|
||||||
self.stop_ticker(session_id)
|
self.stop_ticker(session_id)
|
||||||
|
|
||||||
def live_count(self) -> int:
|
def live_count(self) -> int:
|
||||||
|
|
@ -158,6 +160,18 @@ class SessionHub:
|
||||||
log.warning("продление lease занятия %s не удалось", state.session_id,
|
log.warning("продление lease занятия %s не удалось", state.session_id,
|
||||||
exc_info=True)
|
exc_info=True)
|
||||||
await self.fence(state)
|
await self.fence(state)
|
||||||
|
for state in list(self._pending_adoptions.values()):
|
||||||
|
try:
|
||||||
|
await self.store.commit(state)
|
||||||
|
except SessionLeaseLost:
|
||||||
|
self._pending_adoptions.pop(state.session_id, None)
|
||||||
|
except Exception:
|
||||||
|
log.warning("повторное сохранение занятия %s после takeover не удалось",
|
||||||
|
state.session_id, exc_info=True)
|
||||||
|
else:
|
||||||
|
self._pending_adoptions.pop(state.session_id, None)
|
||||||
|
self.register(state)
|
||||||
|
self.start_ticker(state.session_id)
|
||||||
try:
|
try:
|
||||||
claimed = await self.store.claim_expired()
|
claimed = await self.store.claim_expired()
|
||||||
except Exception: # следующий оборот попробует снова
|
except Exception: # следующий оборот попробует снова
|
||||||
|
|
@ -183,9 +197,13 @@ class SessionHub:
|
||||||
state.socket_connected_at_checkpoint = False
|
state.socket_connected_at_checkpoint = False
|
||||||
try:
|
try:
|
||||||
await self.store.commit(state)
|
await self.store.commit(state)
|
||||||
|
except SessionLeaseLost:
|
||||||
|
return False
|
||||||
except Exception:
|
except Exception:
|
||||||
log.exception("не удалось сохранить присутствие после takeover %s", state.session_id)
|
log.exception("не удалось сохранить присутствие после takeover %s", state.session_id)
|
||||||
|
self._pending_adoptions[state.session_id] = state
|
||||||
return False
|
return False
|
||||||
|
self._pending_adoptions.pop(state.session_id, None)
|
||||||
self.register(state)
|
self.register(state)
|
||||||
self.start_ticker(state.session_id)
|
self.start_ticker(state.session_id)
|
||||||
return True
|
return True
|
||||||
|
|
|
||||||
|
|
@ -461,3 +461,33 @@ def test_failed_presence_commit_closes_station_with_fencing_code(client, monkeyp
|
||||||
assert not hub.station_connected(session_id)
|
assert not hub.station_connected(session_id)
|
||||||
finally:
|
finally:
|
||||||
control.__exit__(None, None, None)
|
control.__exit__(None, None, None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_takeover_retries_failed_presence_checkpoint():
|
||||||
|
class FlakyStore(MemorySessionStore):
|
||||||
|
def __init__(self):
|
||||||
|
self.commits = 0
|
||||||
|
|
||||||
|
async def commit(self, _state, _records=()):
|
||||||
|
self.commits += 1
|
||||||
|
if self.commits == 1:
|
||||||
|
raise OSError("temporary database outage")
|
||||||
|
|
||||||
|
store = FlakyStore()
|
||||||
|
local_hub = SessionHub(store)
|
||||||
|
state = SessionState(
|
||||||
|
session_id=uuid4(), scenario_id="test", scenario_title="Тест", level="L1",
|
||||||
|
mode=SessionMode.TRAINING, exercise=Exercise.DDS,
|
||||||
|
socket_last_seen_at=datetime.now(UTC) - timedelta(minutes=10),
|
||||||
|
socket_connected_at_checkpoint=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def recover():
|
||||||
|
assert not await local_hub._adopt(state)
|
||||||
|
assert local_hub.get(state.session_id) is None
|
||||||
|
await local_hub.maintain_lease()
|
||||||
|
assert local_hub.get(state.session_id) is state
|
||||||
|
assert store.commits == 2
|
||||||
|
await local_hub.shutdown()
|
||||||
|
|
||||||
|
asyncio.run(recover())
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue