import crypto from 'node:crypto'; import { createIdleFailoverSnapshot, isFailoverConfigured, nextFailoverDecision, normalizeFailoverPolicy, type AppliedFailoverPolicy, type FailoverDecisionMemory, type FailoverHealth, type FailoverPolicy, type FailoverRole, type FailoverSnapshot, } from '../../../shared/failover.js'; import type { HarborServer, StoredState } from '../../../shared/contracts/state.js'; import type { ActivityJournalEventInput } from '../../../shared/activityJournal.js'; import { HarborError } from '../../../shared/errors.js'; interface Candidate { config: unknown; applied: AppliedFailoverPolicy; } interface FailoverServiceDependencies { state: { read(): StoredState; update(mutator: (state: StoredState) => Record): StoredState; }; runtime: { isRunning(): Promise }; dataplane: { checkConfig(config: unknown): Promise; runFailoverProbe(role: FailoverRole, services: unknown, target: string, timeoutMs: number): Promise; readFailoverSelector(): Promise; selectFailoverRole(role: FailoverRole): Promise; setFailoverActivityEnabled(enabled: boolean): Promise; readFailoverActivity(thresholdBytesPerSecond: number): Promise; }; buildCandidate(state: StoredState): Candidate; serialize(operation: () => Promise): Promise; scheduler?: { setTimeout(callback: () => void, intervalMs: number): NodeJS.Timeout; clearTimeout(timer: NodeJS.Timeout): void; }; now?: () => Date; onWarning?: (error: unknown) => void; onSwitch?: (from: FailoverRole, to: FailoverRole, reason: string) => void; onEvent?: (event: ActivityJournalEventInput) => void; } const record = (value: unknown): Record => ( value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {} ); function targetServer(state: StoredState, role: FailoverRole): HarborServer | null { const target = state.appliedFailoverPolicy?.[role] || state.failoverPolicy[role]; return state.profiles.find(({ id }) => id === target.profileId) ?.servers.find(({ id }) => id === target.serverId) || null; } function currentRole(state: StoredState): FailoverRole | null { const applied = state.appliedFailoverPolicy; if (!applied) return null; for (const role of ['primary', 'reserve'] as const) { if ( state.appliedProfileId === applied[role].profileId && state.appliedServerId === applied[role].serverId ) return role; } return null; } function probeHealth(value: unknown): boolean { const vpn = record(record(value).vpn); const sites = Array.isArray(vpn.sites) ? vpn.sites.map(record) : []; return sites.length === 1 && sites[0].status === 'available'; } function safeErrorCode(error: unknown) { const code = error && typeof error === 'object' && 'code' in error ? String(error.code) : ''; return /^[A-Z0-9_]{1,50}$/.test(code) ? code : 'UNKNOWN'; } export function createFailoverService(dependencies: FailoverServiceDependencies) { const scheduler = dependencies.scheduler || { setTimeout: (callback: () => void, intervalMs: number) => setTimeout(callback, intervalMs), clearTimeout: (timer: NodeJS.Timeout) => clearTimeout(timer), }; const now = dependencies.now || (() => new Date()); const epoch = crypto.randomUUID(); let sequence = 0; let generation = 0; let timer: NodeJS.Timeout | null = null; let collectorEnabled: boolean | null = null; let roundPromise: Promise | null = null; let roundGeneration: number | null = null; let decisionMemory: FailoverDecisionMemory | undefined; const healthMemory: Record = { primary: undefined, reserve: undefined, }; function clearHealthMemory() { healthMemory.primary = undefined; healthMemory.reserve = undefined; } function recordHealthTransition( role: FailoverRole, health: FailoverHealth, capturedGeneration?: number, ) { if (capturedGeneration !== undefined && capturedGeneration !== generation) return; if (health !== 'healthy' && health !== 'unhealthy') return; const previous = healthMemory[role]; healthMemory[role] = health; if (previous === health) return; if (previous === undefined && health === 'healthy') return; const type = health === 'unhealthy' ? role === 'primary' ? 'failover.primary_unavailable' : 'failover.reserve_unavailable' : role === 'primary' ? 'failover.primary_recovered' : 'failover.reserve_recovered'; dependencies.onEvent?.({ type, severity: health === 'unhealthy' ? 'warning' : 'info', source: 'failover', dedupeKey: null, data: { role, reason: health === 'unhealthy' ? 'probe-failed' : 'probe-recovered', }, }); } let snapshot = createIdleFailoverSnapshot(dependencies.state.read().failoverPolicy, epoch, sequence); function appliedMatchesDesired(state: StoredState) { if (!state.appliedFailoverPolicy) return false; try { return JSON.stringify(dependencies.buildCandidate(state).applied) === JSON.stringify(state.appliedFailoverPolicy); } catch { return false; } } function publish(next: FailoverSnapshot) { sequence += 1; snapshot = { ...next, observationEpoch: epoch, observationSequence: sequence }; } function clearTimer() { if (timer) scheduler.clearTimeout(timer); timer = null; } function schedule(delay: number) { clearTimer(); timer = scheduler.setTimeout(() => { timer = null; void runRound().catch(dependencies.onWarning); }, delay); timer.unref?.(); } async function disableCollector() { if (collectorEnabled === false) return; await dependencies.dataplane.setFailoverActivityEnabled(false); collectorEnabled = false; } async function deactivate(policy: FailoverPolicy, passiveRole: FailoverRole | null = null) { generation += 1; clearTimer(); decisionMemory = undefined; clearHealthMemory(); await disableCollector(); const idle = createIdleFailoverSnapshot(policy, epoch, sequence); if (passiveRole) { idle.activation = 'passive-loaded'; idle.currentRole = passiveRole; idle.reason = 'disabled'; } publish(idle); } function activeSnapshot(state: StoredState, status: FailoverSnapshot['status'] = 'observing'): FailoverSnapshot { const role = state.failoverRuntimeState.reasonCode === 'selector-unknown' ? null : currentRole(state); const channel = (target: typeof state.failoverPolicy.primary) => ({ target, health: 'unknown' as FailoverHealth, failingServiceIds: [], checkedAt: null, stateSince: null, }); return { observationEpoch: epoch, observationSequence: sequence, configured: isFailoverConfigured(state.failoverPolicy), enabled: state.failoverPolicy.enabled, paused: state.failoverPolicy.paused, activation: appliedMatchesDesired(state) ? 'active' : 'pending', currentRole: role || 'other', status, primary: channel(state.appliedFailoverPolicy?.primary || state.failoverPolicy.primary), reserve: channel(state.appliedFailoverPolicy?.reserve || state.failoverPolicy.reserve), nextDecisionAt: null, reason: null, trafficActivity: null, policy: state.failoverPolicy, }; } async function reconcile() { let state = dependencies.state.read(); const policy = state.failoverPolicy; if (!policy.enabled) { const role = currentRole(state); const passiveRole = role && await dependencies.runtime.isRunning() ? role : null; return deactivate(policy, passiveRole); } const running = await dependencies.runtime.isRunning(); if (!running || !state.appliedFailoverPolicy || !currentRole(state)) { generation += 1; clearHealthMemory(); clearTimer(); await disableCollector(); const pending = activeSnapshot(state, 'idle'); pending.activation = running ? 'pending' : 'inactive'; pending.reason = running ? 'pending-activation' : 'vpn-stopped'; publish(pending); return; } const role = currentRole(state)!; const selected = record(await dependencies.dataplane.readFailoverSelector()); if (selected.role !== role) await dependencies.dataplane.selectFailoverRole(role); if (state.failoverRuntimeState.reasonCode === 'selector-unknown') { state = dependencies.state.update((current) => ({ ...current, failoverRuntimeState: { ...current.failoverRuntimeState, reasonCode: null }, })); } if (collectorEnabled !== true) { await dependencies.dataplane.setFailoverActivityEnabled(true); collectorEnabled = true; } generation += 1; const active = activeSnapshot(state); if (active.activation === 'pending') active.reason = 'pending-activation'; publish(active); schedule(0); } async function reconcileAfterCommit() { try { await reconcile(); } catch (error) { dependencies.onWarning?.(error); const failed = activeSnapshot(dependencies.state.read(), 'error'); failed.reason = 'reconcile-failed'; publish(failed); } } async function assessRole(role: FailoverRole, policy: FailoverPolicy) { const custom = dependencies.state.read().diagnostics.customServices; const results = await Promise.all(policy.checks.map(async (check) => { try { return { id: check.serviceId, ok: probeHealth(await dependencies.dataplane.runFailoverProbe( role, custom, `site:${check.serviceId}`, check.timeoutMs, )), }; } catch { return { id: check.serviceId, ok: null }; } })); return { health: results.some(({ ok }) => ok === null) ? 'unknown' as const : results.every(({ ok }) => ok) ? 'healthy' as const : 'unhealthy' as const, failingServiceIds: results.filter(({ ok }) => ok === false).map(({ id }) => id), }; } async function switchWithinQueue(role: FailoverRole, reason: string) { const before = dependencies.state.read(); const from = currentRole(before); if (!from || from === role) return; try { await dependencies.dataplane.selectFailoverRole(role); const server = targetServer(before, role); if (!server) throw new HarborError('SERVER_NOT_FOUND'); const target = before.appliedFailoverPolicy![role]; const switchedAt = now().toISOString(); const cutoff = now().getTime() - before.failoverPolicy.flapProtection.windowMs; const history = role === 'reserve' ? [...before.failoverRuntimeState.failoverHistory.filter((value) => Date.parse(value) >= cutoff), switchedAt] : before.failoverRuntimeState.failoverHistory.filter((value) => Date.parse(value) >= cutoff); const quarantine = history.length >= before.failoverPolicy.flapProtection.count ? new Date(now().getTime() + before.failoverPolicy.flapProtection.quarantineMs).toISOString() : before.failoverRuntimeState.primaryQuarantineUntil; dependencies.state.update((state) => ({ ...state, appliedProfileId: target.profileId, appliedServerId: target.serverId, appliedServerSnapshot: server, failoverPolicy: reason === 'manual' ? state.failoverPolicy : role === 'primary' ? { ...state.failoverPolicy, paused: false } : state.failoverPolicy, failoverRuntimeState: { ...state.failoverRuntimeState, lastSwitchAt: switchedAt, holdUntil: role === 'reserve' ? new Date(now().getTime() + state.failoverPolicy.minimumReserveMs).toISOString() : null, primaryQuarantineUntil: quarantine, failoverHistory: history, reasonCode: reason, }, })); } catch (error) { try { await dependencies.dataplane.selectFailoverRole(from); } catch (rollback) { generation += 1; clearTimer(); decisionMemory = undefined; dependencies.state.update((state) => ({ ...state, failoverPolicy: { ...state.failoverPolicy, paused: true }, failoverRuntimeState: { ...state.failoverRuntimeState, reasonCode: 'selector-unknown' }, })); const failed = activeSnapshot(dependencies.state.read(), 'error'); failed.reason = 'selector-unknown'; publish(failed); throw new AggregateError([error, rollback], 'Failover selector rollback failed'); } throw error; } dependencies.onSwitch?.(from, role, reason); dependencies.onEvent?.({ type: 'failover.switched', severity: 'info', source: 'failover', dedupeKey: `failover.switched:${dependencies.state.read().revision}`, data: { fromRole: from, toRole: role, primaryLabel: targetServer(dependencies.state.read(), 'primary')?.label || 'Primary', reserveLabel: targetServer(dependencies.state.read(), 'reserve')?.label || 'Reserve', reason, manual: reason === 'manual', }, }); } async function performRound(capturedGeneration: number) { const prepared = await dependencies.serialize(async () => { const state = dependencies.state.read(); const role = currentRole(state); if ( capturedGeneration !== generation || !state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !role ) return null; const selected = record(await dependencies.dataplane.readFailoverSelector()); if (selected.role !== role) await dependencies.dataplane.selectFailoverRole(role); await dependencies.dataplane.setFailoverActivityEnabled(true); collectorEnabled = true; const latest = dependencies.state.read(); return capturedGeneration === generation && latest.failoverPolicy.enabled && currentRole(latest) === role ? { state: latest, policy: latest.failoverPolicy, role } : null; }); if (!prepared) return; const { state, policy, role } = prepared; const checkedAt = now().toISOString(); const previousSnapshot = snapshot; publish({ ...snapshot, reason: 'checking-channels' }); let primary; let reserve; try { [primary, reserve] = await Promise.all([ assessRole('primary', policy), assessRole('reserve', policy), ]); } catch { primary = { health: 'unknown' as const, failingServiceIds: [] }; reserve = { health: 'unknown' as const, failingServiceIds: [] }; } if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return; recordHealthTransition('primary', primary.health, capturedGeneration); recordHealthTransition('reserve', reserve.health, capturedGeneration); const activityResponse = record(await dependencies.dataplane.readFailoverActivity( policy.trafficGuard.thresholdBytesPerSecond, )); if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return; const activity = record(activityResponse.activity); const observedAt = typeof activity.observedAt === 'string' ? Date.parse(activity.observedAt) : NaN; const activityState = Number.isFinite(observedAt) && now().getTime() - observedAt <= 4_000 && (activity.state === 'active' || activity.state === 'quiet') ? activity.state : 'unknown'; const decision = nextFailoverDecision({ now: now().getTime(), policy, currentRole: role, primaryHealth: primary.health, reserveHealth: reserve.health, activity: activityState, holdUntil: Date.parse(state.failoverRuntimeState.holdUntil || '') || null, primaryQuarantineUntil: Date.parse(state.failoverRuntimeState.primaryQuarantineUntil || '') || null, memory: decisionMemory, }); decisionMemory = decision.memory; const next = activeSnapshot(state, decision.status); next.currentRole = role; next.primary = { ...next.primary, ...primary, checkedAt, stateSince: previousSnapshot.primary.health === primary.health ? previousSnapshot.primary.stateSince || checkedAt : checkedAt, }; next.reserve = { ...next.reserve, ...reserve, checkedAt, stateSince: previousSnapshot.reserve.health === reserve.health ? previousSnapshot.reserve.stateSince || checkedAt : checkedAt, }; next.reason = decision.reason; next.nextDecisionAt = decision.nextDecisionAt ? new Date(decision.nextDecisionAt).toISOString() : null; next.trafficActivity = activityState === 'unknown' ? { state: 'unknown', observedAt: Number.isFinite(observedAt) ? new Date(observedAt).toISOString() : checkedAt, windowMs: 10_000, thresholdBytesPerSecond: policy.trafficGuard.thresholdBytesPerSecond, totalBytesPerSecond: 0, transmittingConnections: 0, quietSince: null, switchTarget: decision.switchTo, blockers: [], } : { state: activityState, observedAt: String(activity.observedAt), windowMs: Number(activity.windowMs) || 10_000, thresholdBytesPerSecond: policy.trafficGuard.thresholdBytesPerSecond, totalBytesPerSecond: Number(activity.totalBytesPerSecond) || 0, transmittingConnections: Number(activity.transmittingConnections) || 0, quietSince: typeof activity.quietSince === 'string' ? activity.quietSince : null, switchTarget: decision.switchTo, blockers: (Array.isArray(activity.blockers) ? activity.blockers : []).slice(0, 3) as FailoverSnapshot['trafficActivity'] extends infer T ? T extends { blockers: infer B } ? B : never : never, }; publish(next); if (decision.status === 'waiting-for-idle' && previousSnapshot.status !== 'waiting-for-idle') { dependencies.onEvent?.({ type: 'failover.waiting_for_idle', severity: 'info', source: 'failover', dedupeKey: `failover.waiting_for_idle:${state.revision}:${role}:${decision.reason}`, data: { fromRole: role, toRole: decision.switchTo || (role === 'primary' ? 'reserve' : 'primary'), reason: decision.reason }, }); } if (decision.reason === 'both-unhealthy' && previousSnapshot.reason !== 'both-unhealthy') { dependencies.onEvent?.({ type: 'failover.both_unhealthy', severity: 'warning', source: 'failover', dedupeKey: `failover.both_unhealthy:${state.revision}`, data: { reason: decision.reason }, }); } if (decision.switchTo) { try { const switched = await dependencies.serialize(async () => { const current = dependencies.state.read(); if ( capturedGeneration !== generation || !current.failoverPolicy.enabled || current.failoverPolicy.paused || currentRole(current) !== role || !current.appliedFailoverPolicy ) return false; let freshPrimary; let freshReserve; try { [freshPrimary, freshReserve] = await Promise.all([ assessRole('primary', current.failoverPolicy), assessRole('reserve', current.failoverPolicy), ]); } catch { return false; } const healthStillAllowsSwitch = decision.switchTo === 'reserve' ? freshPrimary.health === 'unhealthy' && freshReserve.health === 'healthy' : freshPrimary.health === 'healthy'; if (!healthStillAllowsSwitch || capturedGeneration !== generation) return false; if (current.failoverPolicy.trafficGuard.enabled) { const freshResponse = record(await dependencies.dataplane.readFailoverActivity( current.failoverPolicy.trafficGuard.thresholdBytesPerSecond, )); const freshActivity = record(freshResponse.activity); const freshObservedAt = typeof freshActivity.observedAt === 'string' ? Date.parse(freshActivity.observedAt) : NaN; const freshQuietSince = typeof freshActivity.quietSince === 'string' ? Date.parse(freshActivity.quietSince) : NaN; if ( capturedGeneration !== generation || freshActivity.state !== 'quiet' || !Number.isFinite(freshObservedAt) || now().getTime() - freshObservedAt > 4_000 || !Number.isFinite(freshQuietSince) || now().getTime() - freshQuietSince < current.failoverPolicy.trafficGuard.quietWindowMs ) return false; } await switchWithinQueue(decision.switchTo!, decision.reason); return true; }); if (!switched) { const cancelled = activeSnapshot(dependencies.state.read(), 'observing'); cancelled.reason = 'revalidation-required'; publish(cancelled); decisionMemory = undefined; return; } } catch (error) { const failed = activeSnapshot(dependencies.state.read(), 'error'); failed.reason = dependencies.state.read().failoverRuntimeState.reasonCode === 'selector-unknown' ? 'selector-unknown' : 'switch-failed'; publish(failed); dependencies.onEvent?.({ type: 'failover.switch_failed', severity: 'error', source: 'failover', dedupeKey: `failover.switch_failed:${state.revision}:${role}:${decision.switchTo}`, data: { fromRole: role, toRole: decision.switchTo, reason: decision.reason, errorCode: safeErrorCode(error) }, }); throw error; } if (capturedGeneration !== generation) return; publish(activeSnapshot(dependencies.state.read(), decision.switchTo === 'reserve' ? 'reserve' : 'primary')); decisionMemory = undefined; } } async function runRound(): Promise { if (roundPromise) { const pending = roundPromise; if (roundGeneration === generation) return pending; try { await pending; } catch { // The original caller owns the stale round error; continue with current-generation work. } if (roundPromise && roundPromise !== pending) return roundPromise; return runRound(); } const capturedGeneration = generation; let trackedPromise: Promise; trackedPromise = performRound(capturedGeneration).finally(() => { if (roundPromise === trackedPromise) { roundPromise = null; roundGeneration = null; } if (capturedGeneration === generation && snapshot.reason === 'checking-channels') { const failed = activeSnapshot(dependencies.state.read(), 'error'); failed.reason = 'health-unknown'; publish(failed); } const policy = dependencies.state.read().failoverPolicy; if (capturedGeneration === generation && policy.enabled && dependencies.state.read().appliedFailoverPolicy) { const decisionAt = snapshot.nextDecisionAt ? Date.parse(snapshot.nextDecisionAt) : NaN; const decisionDelay = Number.isFinite(decisionAt) ? Math.max(250, decisionAt - now().getTime()) : policy.intervalMs; schedule(Math.min(policy.intervalMs, decisionDelay)); } }); roundPromise = trackedPromise; roundGeneration = capturedGeneration; return trackedPromise; } function save(value: unknown) { return dependencies.serialize(async () => { let policy: FailoverPolicy; try { policy = normalizeFailoverPolicy(value, { strict: true }); } catch (cause) { throw new HarborError('REQUEST_INVALID', { cause }); } if (policy.enabled && !isFailoverConfigured(policy)) throw new HarborError('REQUEST_INVALID'); const before = dependencies.state.read(); const candidateState = { ...before, failoverPolicy: policy }; if (policy.enabled) { const candidate = dependencies.buildCandidate(candidateState); await dependencies.dataplane.checkConfig(candidate.config); } dependencies.state.update((state) => { const role = before.failoverPolicy.enabled && !policy.enabled ? currentRole(state) : null; const target = role ? state.appliedFailoverPolicy?.[role] : null; return { ...state, failoverPolicy: policy, ...(target ? { desiredProfileId: target.profileId, profiles: state.profiles.map((profile) => profile.id === target.profileId ? { ...profile, desiredServerId: target.serverId } : profile), } : {}), }; }); decisionMemory = undefined; if (before.failoverPolicy.enabled !== policy.enabled) clearHealthMemory(); if (before.failoverPolicy.enabled !== policy.enabled) { dependencies.onEvent?.({ type: policy.enabled ? 'failover.enabled' : 'failover.disabled', severity: 'info', source: 'failover', dedupeKey: `failover.${policy.enabled ? 'enabled' : 'disabled'}:${dependencies.state.read().revision}`, data: {}, }); } await reconcileAfterCommit(); }); } function pause(paused: boolean) { return dependencies.serialize(async () => { const before = dependencies.state.read(); if (!paused && ( !before.appliedFailoverPolicy || !targetServer(before, 'primary') || !targetServer(before, 'reserve') )) throw new HarborError('REQUEST_INVALID'); dependencies.state.update((state) => ({ ...state, failoverPolicy: { ...state.failoverPolicy, paused }, })); decisionMemory = undefined; dependencies.onEvent?.({ type: paused ? 'failover.paused' : 'failover.resumed', severity: 'info', source: 'failover', dedupeKey: `failover.${paused ? 'paused' : 'resumed'}:${dependencies.state.read().revision}`, data: {}, }); await reconcileAfterCommit(); }); } async function manualSwitch(role: FailoverRole) { await dependencies.serialize(async () => { const state = dependencies.state.read(); if (!state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !currentRole(state)) { throw new HarborError('REQUEST_INVALID'); } await switchWithinQueue(role, 'manual'); }); await reconcileAfterCommit(); } const prepareActivation = (role: FailoverRole) => dependencies.dataplane.selectFailoverRole(role); async function restoreAppliedActivation(state: StoredState) { if (!state.appliedFailoverPolicy) return; const role = currentRole(state); if (!role) throw new HarborError('CONFIG_INVALID'); await prepareActivation(role); } function checkNow() { return dependencies.serialize(async () => { const state = dependencies.state.read(); if (!state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !currentRole(state)) { throw new HarborError('REQUEST_INVALID'); } const capturedGeneration = ++generation; clearTimer(); const checkedAt = now().toISOString(); publish({ ...snapshot, reason: 'checking-channels' }); let primary; let reserve; try { [primary, reserve] = await Promise.all([ assessRole('primary', state.failoverPolicy), assessRole('reserve', state.failoverPolicy), ]); } catch { primary = { health: 'unknown' as const, failingServiceIds: [] }; reserve = { health: 'unknown' as const, failingServiceIds: [] }; } if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return; recordHealthTransition('primary', primary.health, capturedGeneration); recordHealthTransition('reserve', reserve.health, capturedGeneration); const next = activeSnapshot(dependencies.state.read(), 'observing'); next.primary = { ...next.primary, ...primary, checkedAt, stateSince: checkedAt }; next.reserve = { ...next.reserve, ...reserve, checkedAt, stateSince: checkedAt }; next.reason = 'manual-check'; publish(next); schedule(state.failoverPolicy.intervalMs); }); } return { snapshot: () => snapshot, save, pause, manualSwitch, checkNow, prepareActivation, restoreAppliedActivation, reconcile, runRound, shutdown: () => deactivate(dependencies.state.read().failoverPolicy), }; } export type FailoverService = ReturnType;