export type FailoverRole = 'primary' | 'reserve'; export type FailoverHealth = 'healthy' | 'unhealthy' | 'unknown' | 'not-monitoring'; export interface FailoverTarget { profileId: string; serverId: string; } export interface FailoverCheck { serviceId: string; timeoutMs: number; } export interface FailoverPolicy { version: 1; enabled: boolean; paused: boolean; primary: FailoverTarget; reserve: FailoverTarget; checks: FailoverCheck[]; intervalMs: number; failureWindowMs: number; recoveryWindowMs: number; trafficGuard: { enabled: boolean; thresholdBytesPerSecond: number; quietWindowMs: number; }; minimumReserveMs: number; flapProtection: { count: number; windowMs: number; quarantineMs: number; }; } export interface FailoverRuntimeState { lastSwitchAt: string | null; holdUntil: string | null; primaryQuarantineUntil: string | null; failoverHistory: string[]; reasonCode: string | null; } export interface AppliedFailoverPolicy { primary: FailoverTarget; reserve: FailoverTarget; primaryConfigFingerprint: string; reserveConfigFingerprint: string; } export interface FailoverActivity { state: 'active' | 'quiet' | 'unknown'; observedAt: string; windowMs: number; thresholdBytesPerSecond: number; totalBytesPerSecond: number; transmittingConnections: number; quietSince: string | null; switchTarget: FailoverRole | null; blockers: Array<{ device: string; service: string; uploadBytesPerSecond: number; downloadBytesPerSecond: number; }>; } export interface FailoverSnapshot { observationEpoch: string; observationSequence: number; configured: boolean; enabled: boolean; paused: boolean; activation: 'inactive' | 'active' | 'pending' | 'passive-loaded'; currentRole: FailoverRole | 'other' | 'none'; status: 'idle' | 'observing' | 'primary' | 'reserve' | 'waiting-for-idle' | 'blocked' | 'switching' | 'error'; primary: { target: FailoverTarget; health: FailoverHealth; failingServiceIds: string[]; checkedAt: string | null; stateSince: string | null }; reserve: { target: FailoverTarget; health: FailoverHealth; failingServiceIds: string[]; checkedAt: string | null; stateSince: string | null }; nextDecisionAt: string | null; reason: string | null; trafficActivity: FailoverActivity | null; policy: FailoverPolicy; } export interface FailoverDecisionMemory { primaryFailedSince: number | null; primaryRecoveredSince: number | null; quietSince: number | null; } export interface FailoverDecisionInput { now: number; policy: FailoverPolicy; currentRole: FailoverRole; primaryHealth: Exclude; reserveHealth: Exclude; activity: 'active' | 'quiet' | 'unknown'; holdUntil?: number | null; primaryQuarantineUntil?: number | null; memory?: FailoverDecisionMemory; } export interface FailoverDecision { status: FailoverSnapshot['status']; reason: string; switchTo: FailoverRole | null; memory: FailoverDecisionMemory; nextDecisionAt: number | null; } const text = (value: unknown) => typeof value === 'string' ? value.trim() : ''; const record = (value: unknown): Record => ( value && typeof value === 'object' && !Array.isArray(value) ? value as Record : {} ); const bounded = (value: unknown, fallback: number, minimum: number, maximum: number) => { const candidate = Number(value); return Number.isFinite(candidate) ? Math.min(maximum, Math.max(minimum, Math.round(candidate))) : fallback; }; const equivalent = (left: unknown, right: unknown): boolean => { if (Array.isArray(left) || Array.isArray(right)) { return Array.isArray(left) && Array.isArray(right) && left.length === right.length && left.every((value, index) => equivalent(value, right[index])); } if (left && right && typeof left === 'object' && typeof right === 'object') { const leftRecord = left as Record; const rightRecord = right as Record; const leftKeys = Object.keys(leftRecord).sort(); const rightKeys = Object.keys(rightRecord).sort(); return leftKeys.length === rightKeys.length && leftKeys.every((key, index) => key === rightKeys[index] && equivalent(leftRecord[key], rightRecord[key])); } return Object.is(left, right); }; export const DEFAULT_FAILOVER_POLICY: FailoverPolicy = Object.freeze({ version: 1, enabled: false, paused: false, primary: { profileId: '', serverId: '' }, reserve: { profileId: '', serverId: '' }, checks: [{ serviceId: 'youtube', timeoutMs: 6_000 }, { serviceId: 'google', timeoutMs: 6_000 }], intervalMs: 60_000, failureWindowMs: 120_000, recoveryWindowMs: 900_000, trafficGuard: { enabled: true, thresholdBytesPerSecond: 32 * 1024, quietWindowMs: 30_000 }, minimumReserveMs: 600_000, flapProtection: { count: 3, windowMs: 86_400_000, quarantineMs: 21_600_000 }, }); export const DEFAULT_FAILOVER_RUNTIME_STATE: FailoverRuntimeState = Object.freeze({ lastSwitchAt: null, holdUntil: null, primaryQuarantineUntil: null, failoverHistory: [], reasonCode: null, }); function target(value: unknown): FailoverTarget { const candidate = record(value); return { profileId: text(candidate.profileId), serverId: text(candidate.serverId) }; } export function normalizeFailoverPolicy(value: unknown, { strict = false } = {}): FailoverPolicy { const candidate = record(value); const requestedChecks = Array.isArray(candidate.checks) ? candidate.checks : DEFAULT_FAILOVER_POLICY.checks; const checks = requestedChecks.map((value) => { const check = record(value); return { serviceId: text(check.serviceId).slice(0, 100), timeoutMs: bounded(check.timeoutMs, 6_000, 2_000, 30_000), }; }).filter(({ serviceId }, index, all) => serviceId && all.findIndex((item) => item.serviceId === serviceId) === index).slice(0, 10); const trafficGuard = record(candidate.trafficGuard); const flapProtection = record(candidate.flapProtection); const policy: FailoverPolicy = { version: 1, enabled: candidate.enabled === true, paused: candidate.paused === true, primary: target(candidate.primary), reserve: target(candidate.reserve), checks, intervalMs: bounded(candidate.intervalMs, 60_000, 15_000, 900_000), failureWindowMs: 0, recoveryWindowMs: bounded(candidate.recoveryWindowMs, 900_000, 60_000, 86_400_000), trafficGuard: { enabled: trafficGuard.enabled !== false, thresholdBytesPerSecond: bounded(trafficGuard.thresholdBytesPerSecond, 32 * 1024, 1024, 100 * 1024 * 1024), quietWindowMs: bounded(trafficGuard.quietWindowMs, 30_000, 5_000, 600_000), }, minimumReserveMs: bounded(candidate.minimumReserveMs, 600_000, 60_000, 86_400_000), flapProtection: { count: bounded(flapProtection.count, 3, 2, 10), windowMs: bounded(flapProtection.windowMs, 86_400_000, 3_600_000, 72 * 3_600_000), quarantineMs: bounded(flapProtection.quarantineMs, 21_600_000, 600_000, 7 * 86_400_000), }, }; policy.failureWindowMs = bounded( candidate.failureWindowMs, 120_000, policy.intervalMs * 2, 1_800_000, ); if (strict && !equivalent(policy, value)) { throw new TypeError('Invalid failover policy'); } return policy; } export function normalizeFailoverRuntimeState(value: unknown): FailoverRuntimeState { const candidate = record(value); const date = (value: unknown) => typeof value === 'string' && Number.isFinite(Date.parse(value)) ? value : null; return { lastSwitchAt: date(candidate.lastSwitchAt), holdUntil: date(candidate.holdUntil), primaryQuarantineUntil: date(candidate.primaryQuarantineUntil), failoverHistory: (Array.isArray(candidate.failoverHistory) ? candidate.failoverHistory : []) .map(date).filter((item): item is string => Boolean(item)).slice(-20), reasonCode: text(candidate.reasonCode) || null, }; } export function normalizeAppliedFailoverPolicy(value: unknown): AppliedFailoverPolicy | null { if (!value) return null; const candidate = record(value); const primary = target(candidate.primary); const reserve = target(candidate.reserve); const primaryConfigFingerprint = text(candidate.primaryConfigFingerprint); const reserveConfigFingerprint = text(candidate.reserveConfigFingerprint); return primary.profileId && primary.serverId && reserve.profileId && reserve.serverId && /^[a-f0-9]{64}$/.test(primaryConfigFingerprint) && /^[a-f0-9]{64}$/.test(reserveConfigFingerprint) ? { primary, reserve, primaryConfigFingerprint, reserveConfigFingerprint } : null; } export function isFailoverConfigured(policy: FailoverPolicy) { return Boolean( policy.primary.profileId && policy.primary.serverId && policy.reserve.profileId && policy.reserve.serverId && policy.checks.length && (policy.primary.profileId !== policy.reserve.profileId || policy.primary.serverId !== policy.reserve.serverId) ); } export function createIdleFailoverSnapshot( policy: FailoverPolicy, observationEpoch = '', observationSequence = 0, ): FailoverSnapshot { const channel = (target: FailoverTarget) => ({ target, health: 'not-monitoring' as const, failingServiceIds: [], checkedAt: null, stateSince: null, }); return { observationEpoch, observationSequence, configured: isFailoverConfigured(policy), enabled: policy.enabled, paused: policy.paused, activation: 'inactive', currentRole: 'none', status: 'idle', primary: channel(policy.primary), reserve: channel(policy.reserve), nextDecisionAt: null, reason: null, trafficActivity: null, policy, }; } export function nextFailoverDecision(input: FailoverDecisionInput): FailoverDecision { const memory = input.memory || { primaryFailedSince: null, primaryRecoveredSince: null, quietSince: null, }; const next = { ...memory }; if (!input.policy.enabled || input.policy.paused) { return { status: 'idle', reason: input.policy.paused ? 'paused' : 'disabled', switchTo: null, memory: next, nextDecisionAt: null }; } if (input.primaryHealth === 'unhealthy' && input.reserveHealth === 'unhealthy') { next.quietSince = null; return { status: 'blocked', reason: 'both-unhealthy', switchTo: null, memory: next, nextDecisionAt: null }; } let target: FailoverRole | null = null; let readyAt: number | null = null; if (input.currentRole === 'primary') { next.primaryRecoveredSince = null; if (input.primaryHealth !== 'unhealthy') { next.primaryFailedSince = null; next.quietSince = null; return { status: 'primary', reason: input.primaryHealth === 'healthy' ? 'primary-healthy' : 'health-unknown', switchTo: null, memory: next, nextDecisionAt: null }; } next.primaryFailedSince ??= input.now; readyAt = next.primaryFailedSince + input.policy.failureWindowMs; if (input.now < readyAt || input.reserveHealth !== 'healthy') { next.quietSince = null; return { status: 'observing', reason: input.reserveHealth === 'healthy' ? 'failure-window' : 'reserve-not-healthy', switchTo: null, memory: next, nextDecisionAt: readyAt }; } target = 'reserve'; } else { next.primaryFailedSince = null; if (input.primaryHealth !== 'healthy') { next.primaryRecoveredSince = null; next.quietSince = null; return { status: 'reserve', reason: 'primary-not-recovered', switchTo: null, memory: next, nextDecisionAt: null }; } if ((input.primaryQuarantineUntil || 0) > input.now) { next.primaryRecoveredSince = null; next.quietSince = null; return { status: 'reserve', reason: 'recovery-hold', switchTo: null, memory: next, nextDecisionAt: input.primaryQuarantineUntil || null, }; } next.primaryRecoveredSince ??= input.now; const recoveredAt = next.primaryRecoveredSince + input.policy.recoveryWindowMs; readyAt = input.reserveHealth === 'unhealthy' ? recoveredAt : Math.max(recoveredAt, input.holdUntil || 0, input.primaryQuarantineUntil || 0); if (input.now < readyAt) { next.quietSince = null; return { status: 'reserve', reason: 'recovery-hold', switchTo: null, memory: next, nextDecisionAt: readyAt }; } target = 'primary'; } if (input.policy.trafficGuard.enabled) { if (input.activity === 'unknown') { next.quietSince = null; return { status: 'blocked', reason: 'activity-unknown', switchTo: null, memory: next, nextDecisionAt: null }; } if (input.activity === 'active') { next.quietSince = null; return { status: 'waiting-for-idle', reason: 'active-traffic', switchTo: null, memory: next, nextDecisionAt: null }; } next.quietSince ??= input.now; readyAt = next.quietSince + input.policy.trafficGuard.quietWindowMs; if (input.now < readyAt) { return { status: 'waiting-for-idle', reason: 'quiet-window', switchTo: null, memory: next, nextDecisionAt: readyAt }; } } return { status: 'switching', reason: target === 'reserve' ? 'primary-failed' : 'primary-recovered', switchTo: target, memory: next, nextDecisionAt: null }; }