import assert from 'node:assert/strict'; import fs from 'node:fs'; import test from 'node:test'; import { DEFAULT_FAILOVER_POLICY, isFailoverConfigured, nextFailoverDecision, normalizeFailoverPolicy, } from '../../dist/shared/failover.js'; import { createFailoverService } from '../../dist/server/features/failover/failoverService.js'; const enabled = normalizeFailoverPolicy({ ...DEFAULT_FAILOVER_POLICY, enabled: true, primary: { profileId: 'profile-1', serverId: 'server-1' }, reserve: { profileId: 'profile-2', serverId: 'server-2' }, intervalMs: 15_000, failureWindowMs: 30_000, recoveryWindowMs: 60_000, trafficGuard: { enabled: true, thresholdBytesPerSecond: 1024, quietWindowMs: 5_000 }, minimumReserveMs: 60_000, }); test('failover policy defaults to strict disabled zero-work state', () => { const policy = normalizeFailoverPolicy(null); const decision = nextFailoverDecision({ now: 0, policy, currentRole: 'primary', primaryHealth: 'unhealthy', reserveHealth: 'healthy', activity: 'quiet', }); assert.equal(policy.enabled, false); assert.equal(decision.status, 'idle'); assert.equal(decision.switchTo, null); }); test('failover save rejects policy values that would otherwise be silently clamped', async () => { const testHarness = harness(serviceState()); await assert.rejects(testHarness.service.save({ ...enabled, intervalMs: -1 }), { code: 'REQUEST_INVALID' }); }); test('strict failover validation accepts semantically identical reordered JSON keys', () => { const reordered = { flapProtection: enabled.flapProtection, minimumReserveMs: enabled.minimumReserveMs, trafficGuard: enabled.trafficGuard, recoveryWindowMs: enabled.recoveryWindowMs, failureWindowMs: enabled.failureWindowMs, intervalMs: enabled.intervalMs, checks: enabled.checks, reserve: enabled.reserve, primary: enabled.primary, paused: enabled.paused, enabled: enabled.enabled, version: enabled.version, }; assert.deepEqual(normalizeFailoverPolicy(reordered, { strict: true }), enabled); }); test('enabled failover requires two distinct existing targets', async () => { const sameTarget = normalizeFailoverPolicy({ ...enabled, reserve: enabled.primary }); assert.equal(isFailoverConfigured(sameTarget), false); await assert.rejects(harness(serviceState()).service.save(sameTarget), { code: 'REQUEST_INVALID' }); const state = serviceState({ ...enabled, paused: true }); state.profiles[1].servers = []; const testHarness = harness(state); await assert.rejects(testHarness.service.pause(false), { code: 'REQUEST_INVALID' }); assert.equal(testHarness.read().failoverPolicy.paused, true); }); test('local Gateway activity uses the existing domain traffic collector only while enabled', () => { const source = fs.readFileSync(new URL('../../src/server/index.ts', import.meta.url), 'utf8'); assert.match(source, /const localFailoverTraffic = !remoteDataplane && settings\.appMode === 'gateway'[\s\S]*createDomainTrafficService/); assert.match(source, /setLocalFailoverActivityEnabled[\s\S]*enableActivity\(\)[\s\S]*setInterval\(refresh, 2_000\)/); assert.match(source, /if \(!enabled\)[\s\S]*clearInterval\(localFailoverTrafficTimer\)[\s\S]*disableActivity\(\)/); assert.doesNotMatch(source, /setFailoverActivityEnabled: async \(\) => \(\{\}\)/); }); test('failure window and traffic guard delay reserve switch until continuous quiet', () => { let decision = nextFailoverDecision({ now: 0, policy: enabled, currentRole: 'primary', primaryHealth: 'unhealthy', reserveHealth: 'healthy', activity: 'active', }); decision = nextFailoverDecision({ now: 30_000, policy: enabled, currentRole: 'primary', primaryHealth: 'unhealthy', reserveHealth: 'healthy', activity: 'active', memory: decision.memory, }); assert.equal(decision.status, 'waiting-for-idle'); decision = nextFailoverDecision({ now: 31_000, policy: enabled, currentRole: 'primary', primaryHealth: 'unhealthy', reserveHealth: 'healthy', activity: 'quiet', memory: decision.memory, }); assert.equal(decision.switchTo, null); decision = nextFailoverDecision({ now: 36_000, policy: enabled, currentRole: 'primary', primaryHealth: 'unhealthy', reserveHealth: 'healthy', activity: 'quiet', memory: decision.memory, }); assert.equal(decision.switchTo, 'reserve'); }); test('both unhealthy and unknown activity never cause a switch', () => { const both = nextFailoverDecision({ now: 30_000, policy: enabled, currentRole: 'primary', primaryHealth: 'unhealthy', reserveHealth: 'unhealthy', activity: 'quiet', }); assert.equal(both.reason, 'both-unhealthy'); assert.equal(both.switchTo, null); const unknown = nextFailoverDecision({ now: 30_000, policy: enabled, currentRole: 'primary', primaryHealth: 'unhealthy', reserveHealth: 'healthy', activity: 'unknown', memory: { primaryFailedSince: 0, primaryRecoveredSince: null, quietSince: null }, }); assert.equal(unknown.reason, 'activity-unknown'); assert.equal(unknown.switchTo, null); }); test('failback waits for recovery, hold and quarantine deadlines', () => { let decision = nextFailoverDecision({ now: 10_000, policy: enabled, currentRole: 'reserve', primaryHealth: 'healthy', reserveHealth: 'healthy', activity: 'quiet', holdUntil: 50_000, primaryQuarantineUntil: 70_000, }); assert.equal(decision.nextDecisionAt, 70_000); decision = nextFailoverDecision({ now: 70_000, policy: enabled, currentRole: 'reserve', primaryHealth: 'healthy', reserveHealth: 'healthy', activity: 'quiet', holdUntil: 50_000, primaryQuarantineUntil: 70_000, memory: decision.memory, }); assert.equal(decision.switchTo, null); assert.equal(decision.nextDecisionAt, 130_000); decision = nextFailoverDecision({ now: 130_000, policy: enabled, currentRole: 'reserve', primaryHealth: 'healthy', reserveHealth: 'healthy', activity: 'quiet', holdUntil: 50_000, primaryQuarantineUntil: 70_000, memory: { ...decision.memory, quietSince: 125_000 }, }); assert.equal(decision.switchTo, 'primary'); const deadReserve = nextFailoverDecision({ now: 70_000, policy: enabled, currentRole: 'reserve', primaryHealth: 'healthy', reserveHealth: 'unhealthy', activity: 'quiet', holdUntil: 500_000, primaryQuarantineUntil: 700_000, memory: { primaryFailedSince: null, primaryRecoveredSince: 10_000, quietSince: 65_000 }, }); assert.equal(deadReserve.switchTo, null); assert.equal(deadReserve.nextDecisionAt, 700_000); }); function serviceState(policy = enabled) { return { revision: 1, failoverPolicy: policy, failoverRuntimeState: { lastSwitchAt: null, holdUntil: null, primaryQuarantineUntil: null, failoverHistory: [], reasonCode: null }, appliedFailoverPolicy: { primary: { profileId: 'profile-1', serverId: 'server-1' }, reserve: { profileId: 'profile-2', serverId: 'server-2' }, primaryConfigFingerprint: 'a'.repeat(64), reserveConfigFingerprint: 'b'.repeat(64), }, appliedProfileId: 'profile-1', appliedServerId: 'server-1', profiles: [ { id: 'profile-1', servers: [{ id: 'server-1', label: 'Primary' }] }, { id: 'profile-2', servers: [{ id: 'server-2', label: 'Reserve' }] }, ], diagnostics: { customServices: [] }, }; } function harness(initial = serviceState(), overrides = {}) { let state = structuredClone(initial); const calls = []; const dependencies = { state: { read: () => state, update: (mutator) => { state = { ...mutator(state), revision: state.revision + 1 }; calls.push('state:update'); return state; }, }, runtime: { isRunning: async () => true }, dataplane: { checkConfig: async () => ({}), runFailoverProbe: async () => ({ vpn: { sites: [{ status: 'available' }] } }), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => { calls.push(`select:${role}`); return { role }; }, setFailoverActivityEnabled: async (value) => { calls.push(`activity:${value}`); }, readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }), }, buildCandidate: () => ({ config: {}, applied: initial.appliedFailoverPolicy }), serialize: (operation) => operation(), scheduler: { setTimeout: () => ({ unref() {} }), clearTimeout: () => {}, }, ...overrides, }; return { service: createFailoverService(dependencies), calls, read: () => state }; } test('disabled failover performs one restart-safe collector cleanup and no steady-state work', async () => { const state = serviceState(normalizeFailoverPolicy(null)); state.appliedFailoverPolicy = null; const testHarness = harness(state); await testHarness.service.reconcile(); assert.deepEqual(testHarness.calls, ['activity:false']); await testHarness.service.reconcile(); assert.deepEqual(testHarness.calls, ['activity:false']); assert.equal(testHarness.service.snapshot().activation, 'inactive'); }); test('disabled failover reports an already loaded dual config without managing it', async () => { const testHarness = harness(serviceState(normalizeFailoverPolicy(null))); await testHarness.service.reconcile(); assert.deepEqual(testHarness.calls, ['activity:false']); assert.equal(testHarness.service.snapshot().activation, 'passive-loaded'); assert.equal(testHarness.service.snapshot().currentRole, 'primary'); }); test('runtime rollback restores the role from applied truth even while disabled', async () => { const state = serviceState(normalizeFailoverPolicy(null)); state.appliedProfileId = state.appliedFailoverPolicy.reserve.profileId; state.appliedServerId = state.appliedFailoverPolicy.reserve.serverId; const testHarness = harness(state); await testHarness.service.restoreAppliedActivation(state); assert.deepEqual(testHarness.calls, ['select:reserve']); }); test('an in-flight observation is discarded after failover is disabled', async () => { const pending = []; const testHarness = harness(serviceState(), { dataplane: { checkConfig: async () => ({}), runFailoverProbe: (role) => new Promise((resolve) => pending.push({ role, resolve })), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; }, setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); }, readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }), }, }); await testHarness.service.reconcile(); const round = testHarness.service.runRound(); await new Promise((resolve) => setImmediate(resolve)); await testHarness.service.save({ ...enabled, enabled: false }); for (const probe of pending) probe.resolve({ vpn: { sites: [{ status: probe.role === 'reserve' ? 'available' : 'unavailable' }] } }); await round; assert.equal(testHarness.read().failoverPolicy.enabled, false); assert.equal(testHarness.calls.some((call) => call.startsWith('select:reserve')), false); assert.deepEqual(testHarness.calls.filter((call) => call.startsWith('activity:')), ['activity:true', 'activity:true', 'activity:false']); }); test('one channel probe failure does not erase the other channel health', async () => { const testHarness = harness(serviceState(), { dataplane: { checkConfig: async () => ({}), runFailoverProbe: async (role) => { if (role === 'primary') throw new Error('primary probe transport failed'); return { vpn: { sites: [{ status: 'available' }] } }; }, readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => ({ role }), setFailoverActivityEnabled: async () => {}, readFailoverActivity: async () => ({ activity: { state: 'active', observedAt: new Date().toISOString() } }), }, }); await testHarness.service.checkNow(); assert.equal(testHarness.service.snapshot().primary.health, 'unknown'); assert.equal(testHarness.service.snapshot().reserve.health, 'healthy'); }); test('monitoring wakes at an earlier decision deadline instead of waiting a full interval', async () => { let clock = 0; const delays = []; const testHarness = harness(serviceState(), { now: () => new Date(clock), scheduler: { setTimeout: (_callback, delay) => { delays.push(delay); return { unref() {} }; }, clearTimeout: () => {}, }, dataplane: { checkConfig: async () => ({}), runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => ({ role }), setFailoverActivityEnabled: async () => {}, readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date(clock).toISOString() } }), }, }); await testHarness.service.reconcile(); await testHarness.service.runRound(); clock = 30_000; await testHarness.service.runRound(); assert.equal(delays.at(-1), enabled.trafficGuard.quietWindowMs); }); test('a late activity response cannot switch after failover is disabled', async () => { let resolveActivity; const activity = new Promise((resolve) => { resolveActivity = resolve; }); const testHarness = harness(serviceState(), { dataplane: { checkConfig: async () => ({}), runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; }, setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); }, readFailoverActivity: async () => activity, }, }); await testHarness.service.reconcile(); const round = testHarness.service.runRound(); await new Promise(setImmediate); await testHarness.service.save({ ...enabled, enabled: false }); resolveActivity({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }); await round; assert.equal(testHarness.calls.some((value) => value.startsWith('select:')), false); }); test('automatic switch revalidates activity immediately before selector mutation', async () => { let clock = 0; let activityReads = 0; const testHarness = harness(serviceState(), { now: () => new Date(clock), dataplane: { checkConfig: async () => ({}), runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; }, setFailoverActivityEnabled: async () => {}, readFailoverActivity: async () => ({ activity: { state: ++activityReads >= 4 ? 'active' : 'quiet', observedAt: new Date(clock).toISOString(), }, }), }, }); await testHarness.service.reconcile(); await testHarness.service.runRound(); clock = 30_000; await testHarness.service.runRound(); clock = 35_000; await testHarness.service.runRound(); assert.equal(testHarness.calls.some((value) => value.startsWith('select:')), false); assert.equal(testHarness.service.snapshot().reason, 'revalidation-required'); }); test('selector read-back failure rolls an uncertain switch back to the canonical role', async () => { const testHarness = harness(serviceState(), { dataplane: { checkConfig: async () => ({}), runFailoverProbe: async () => ({}), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); if (role === 'reserve') throw new Error('read-back failed'); return { role }; }, setFailoverActivityEnabled: async () => {}, readFailoverActivity: async () => ({ activity: null }), }, }); await assert.rejects(testHarness.service.manualSwitch('reserve'), /read-back failed/); assert.deepEqual(testHarness.calls, ['select:reserve', 'select:primary']); assert.equal(testHarness.read().appliedServerId, 'server-1'); }); test('failed collector disable is retried because local state remains unknown', async () => { let attempts = 0; const testHarness = harness(serviceState(normalizeFailoverPolicy(null)), { dataplane: { checkConfig: async () => ({}), runFailoverProbe: async () => ({}), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async () => ({}), setFailoverActivityEnabled: async () => { attempts += 1; if (attempts === 1) throw new Error('dataplane unavailable'); }, readFailoverActivity: async () => ({ activity: null }), }, }); await assert.rejects(testHarness.service.reconcile(), /dataplane unavailable/); await testHarness.service.reconcile(); assert.equal(attempts, 2); }); test('each monitoring round repairs selector and collector state after a dataplane restart', async () => { const state = serviceState(); state.appliedProfileId = 'profile-2'; state.appliedServerId = 'server-2'; let selected = 'reserve'; let enables = 0; const testHarness = harness(state, { dataplane: { checkConfig: async () => ({}), runFailoverProbe: async () => ({ vpn: { sites: [{ status: 'available' }] } }), readFailoverSelector: async () => ({ role: selected }), selectFailoverRole: async (role) => { selected = role; return { role }; }, setFailoverActivityEnabled: async (value) => { if (value) enables += 1; }, readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }), }, }); await testHarness.service.reconcile(); selected = 'primary'; await testHarness.service.runRound(); assert.equal(selected, 'reserve'); assert.equal(enables, 2); }); test('selector acknowledgement precedes state commit and a failed commit rolls selector back', async () => { const state = serviceState(); const calls = []; const testHarness = harness(state, { state: { read: () => state, update: (mutator) => { const candidate = mutator(state); calls.push('state:update'); if (candidate.appliedServerId === 'server-2') throw new Error('write failed'); Object.assign(state, candidate); return state; }, }, dataplane: { checkConfig: async () => ({}), runFailoverProbe: async () => ({}), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => { calls.push(`select:${role}`); return { role }; }, setFailoverActivityEnabled: async () => {}, readFailoverActivity: async () => ({ activity: null }), }, }); await assert.rejects(testHarness.service.manualSwitch('reserve'), /write failed/); assert.deepEqual(calls, ['select:reserve', 'state:update', 'select:primary']); assert.equal(state.appliedServerId, 'server-1'); }); test('manual selector and pause publish in one canonical commit', async () => { const testHarness = harness(serviceState()); await testHarness.service.manualSwitch('reserve'); assert.equal(testHarness.read().appliedServerId, 'server-2'); assert.equal(testHarness.read().failoverPolicy.paused, true); assert.equal(testHarness.calls.filter((call) => call === 'state:update').length, 1); }); test('post-commit monitoring failure does not report a committed pause as failed', async () => { const warnings = []; const testHarness = harness(serviceState(), { runtime: { isRunning: async () => { throw new Error('runtime unavailable'); } }, onWarning: (error) => warnings.push(error.message), }); await testHarness.service.pause(true); assert.equal(testHarness.read().failoverPolicy.paused, true); assert.equal(testHarness.service.snapshot().status, 'error'); assert.deepEqual(warnings, ['runtime unavailable']); }); test('pause and resume discard stale failure-window evidence', async () => { let clock = 1_000; const testHarness = harness(serviceState(), { now: () => new Date(clock), dataplane: { checkConfig: async () => ({}), runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] }, }), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; }, setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); }, readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date(clock).toISOString(), quietSince: new Date(clock - 10_000).toISOString() }, }), }, }); await testHarness.service.runRound(); await testHarness.service.pause(true); clock = 3_600_000; await testHarness.service.pause(false); await testHarness.service.runRound(); assert.equal(testHarness.calls.includes('select:reserve'), false); assert.equal(testHarness.service.snapshot().reason, 'failure-window'); }); test('an unconfirmed selector rollback durably pauses automation', async () => { const testHarness = harness(serviceState(), { dataplane: { checkConfig: async () => ({}), runFailoverProbe: async () => ({}), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async () => { throw new Error('selector unavailable'); }, setFailoverActivityEnabled: async () => {}, readFailoverActivity: async () => ({ activity: null }), }, }); await assert.rejects(testHarness.service.manualSwitch('reserve'), AggregateError); assert.equal(testHarness.read().failoverPolicy.paused, true); assert.equal(testHarness.read().failoverRuntimeState.reasonCode, 'selector-unknown'); assert.equal(testHarness.service.snapshot().reason, 'selector-unknown'); assert.equal(testHarness.service.snapshot().currentRole, 'other'); }); test('successful selector reconciliation clears an unknown-role latch', async () => { const state = serviceState(); state.failoverRuntimeState.reasonCode = 'selector-unknown'; const testHarness = harness(state); await testHarness.service.reconcile(); assert.equal(testHarness.read().failoverRuntimeState.reasonCode, null); assert.equal(testHarness.service.snapshot().currentRole, 'primary'); }); test('disabling on reserve preserves it as the next ordinary single-channel target', async () => { const state = serviceState(); state.appliedProfileId = 'profile-2'; state.appliedServerId = 'server-2'; const testHarness = harness(state); await testHarness.service.save({ ...enabled, enabled: false }); assert.equal(testHarness.read().desiredProfileId, 'profile-2'); assert.equal(testHarness.read().profiles[1].desiredServerId, 'server-2'); assert.equal(testHarness.service.snapshot().activation, 'passive-loaded'); }); test('significant health states append once without logging every observation', async () => { let clock = 0; const health = { primary: 'unavailable', reserve: 'available' }; const events = []; const testHarness = harness(serviceState(), { now: () => new Date(clock), onEvent: (event) => events.push(event), dataplane: { checkConfig: async () => ({}), runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: health[role] }] } }), readFailoverSelector: async () => ({ role: 'primary' }), selectFailoverRole: async (role) => ({ role }), setFailoverActivityEnabled: async () => {}, readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date(clock).toISOString() } }), }, }); await testHarness.service.reconcile(); await testHarness.service.runRound(); clock = 30_000; await testHarness.service.runRound(); health.reserve = 'unavailable'; clock = 31_000; await testHarness.service.runRound(); clock = 32_000; await testHarness.service.runRound(); health.primary = 'available'; clock = 33_000; await testHarness.service.runRound(); assert.deepEqual(events.map(({ type }) => type), [ 'failover.waiting_for_idle', 'failover.both_unhealthy', 'failover.recovered', ]); }); test('pending target edits keep selector commits bound to the loaded channel pair', async () => { const state = serviceState(); state.failoverPolicy = normalizeFailoverPolicy({ ...state.failoverPolicy, reserve: { profileId: 'profile-3', serverId: 'server-3' }, }); state.profiles.push({ id: 'profile-3', servers: [{ id: 'server-3', label: 'Desired later' }] }); const testHarness = harness(state, { buildCandidate: () => ({ config: {}, applied: { ...state.appliedFailoverPolicy, reserve: { profileId: 'profile-3', serverId: 'server-3' }, reserveConfigFingerprint: 'c'.repeat(64), }, }), }); await testHarness.service.manualSwitch('reserve'); assert.equal(testHarness.read().appliedProfileId, 'profile-2'); assert.equal(testHarness.read().appliedServerId, 'server-2'); assert.equal(testHarness.service.snapshot().activation, 'pending'); assert.deepEqual(testHarness.service.snapshot().reserve.target, { profileId: 'profile-2', serverId: 'server-2' }); });