801 lines
32 KiB
JavaScript
801 lines
32 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import test from 'node:test';
|
|
|
|
import {
|
|
DEFAULT_FAILOVER_POLICY,
|
|
isFailoverConfigured,
|
|
nextFailoverDecision,
|
|
normalizeFailoverPolicy,
|
|
} from '../../dist/shared/failover.js';
|
|
import { createFailoverService } from '../../dist/server/features/failover/failoverService.js';
|
|
|
|
const enabled = normalizeFailoverPolicy({
|
|
...DEFAULT_FAILOVER_POLICY,
|
|
enabled: true,
|
|
primary: { profileId: 'profile-1', serverId: 'server-1' },
|
|
reserve: { profileId: 'profile-2', serverId: 'server-2' },
|
|
intervalMs: 15_000,
|
|
failureWindowMs: 30_000,
|
|
recoveryWindowMs: 60_000,
|
|
trafficGuard: { enabled: true, thresholdBytesPerSecond: 1024, quietWindowMs: 5_000 },
|
|
minimumReserveMs: 60_000,
|
|
});
|
|
|
|
test('failover policy defaults to strict disabled zero-work state', () => {
|
|
const policy = normalizeFailoverPolicy(null);
|
|
const decision = nextFailoverDecision({
|
|
now: 0,
|
|
policy,
|
|
currentRole: 'primary',
|
|
primaryHealth: 'unhealthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'quiet',
|
|
});
|
|
assert.equal(policy.enabled, false);
|
|
assert.equal(decision.status, 'idle');
|
|
assert.equal(decision.switchTo, null);
|
|
});
|
|
|
|
test('failover save rejects policy values that would otherwise be silently clamped', async () => {
|
|
const testHarness = harness(serviceState());
|
|
await assert.rejects(testHarness.service.save({ ...enabled, intervalMs: -1 }), { code: 'REQUEST_INVALID' });
|
|
});
|
|
|
|
test('strict failover validation accepts semantically identical reordered JSON keys', () => {
|
|
const reordered = {
|
|
flapProtection: enabled.flapProtection,
|
|
minimumReserveMs: enabled.minimumReserveMs,
|
|
trafficGuard: enabled.trafficGuard,
|
|
recoveryWindowMs: enabled.recoveryWindowMs,
|
|
failureWindowMs: enabled.failureWindowMs,
|
|
intervalMs: enabled.intervalMs,
|
|
checks: enabled.checks,
|
|
reserve: enabled.reserve,
|
|
primary: enabled.primary,
|
|
paused: enabled.paused,
|
|
enabled: enabled.enabled,
|
|
version: enabled.version,
|
|
};
|
|
assert.deepEqual(normalizeFailoverPolicy(reordered, { strict: true }), enabled);
|
|
});
|
|
|
|
test('enabled failover requires two distinct existing targets', async () => {
|
|
const sameTarget = normalizeFailoverPolicy({ ...enabled, reserve: enabled.primary });
|
|
assert.equal(isFailoverConfigured(sameTarget), false);
|
|
await assert.rejects(harness(serviceState()).service.save(sameTarget), { code: 'REQUEST_INVALID' });
|
|
|
|
const state = serviceState({ ...enabled, paused: true });
|
|
state.profiles[1].servers = [];
|
|
const testHarness = harness(state);
|
|
await assert.rejects(testHarness.service.pause(false), { code: 'REQUEST_INVALID' });
|
|
assert.equal(testHarness.read().failoverPolicy.paused, true);
|
|
});
|
|
|
|
test('local Gateway activity uses the existing domain traffic collector only while enabled', () => {
|
|
const source = fs.readFileSync(new URL('../../src/server/index.ts', import.meta.url), 'utf8');
|
|
assert.match(source, /const localFailoverTraffic = !remoteDataplane && settings\.appMode === 'gateway'[\s\S]*createDomainTrafficService/);
|
|
assert.match(source, /setLocalFailoverActivityEnabled[\s\S]*enableActivity\(\)[\s\S]*setInterval\(refresh, 2_000\)/);
|
|
assert.match(source, /if \(!enabled\)[\s\S]*clearInterval\(localFailoverTrafficTimer\)[\s\S]*disableActivity\(\)/);
|
|
assert.doesNotMatch(source, /setFailoverActivityEnabled: async \(\) => \(\{\}\)/);
|
|
});
|
|
|
|
test('failure window and traffic guard delay reserve switch until continuous quiet', () => {
|
|
let decision = nextFailoverDecision({
|
|
now: 0,
|
|
policy: enabled,
|
|
currentRole: 'primary',
|
|
primaryHealth: 'unhealthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'active',
|
|
});
|
|
decision = nextFailoverDecision({
|
|
now: 30_000,
|
|
policy: enabled,
|
|
currentRole: 'primary',
|
|
primaryHealth: 'unhealthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'active',
|
|
memory: decision.memory,
|
|
});
|
|
assert.equal(decision.status, 'waiting-for-idle');
|
|
decision = nextFailoverDecision({
|
|
now: 31_000,
|
|
policy: enabled,
|
|
currentRole: 'primary',
|
|
primaryHealth: 'unhealthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'quiet',
|
|
memory: decision.memory,
|
|
});
|
|
assert.equal(decision.switchTo, null);
|
|
decision = nextFailoverDecision({
|
|
now: 36_000,
|
|
policy: enabled,
|
|
currentRole: 'primary',
|
|
primaryHealth: 'unhealthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'quiet',
|
|
memory: decision.memory,
|
|
});
|
|
assert.equal(decision.switchTo, 'reserve');
|
|
});
|
|
|
|
test('both unhealthy and unknown activity never cause a switch', () => {
|
|
const both = nextFailoverDecision({
|
|
now: 30_000,
|
|
policy: enabled,
|
|
currentRole: 'primary',
|
|
primaryHealth: 'unhealthy',
|
|
reserveHealth: 'unhealthy',
|
|
activity: 'quiet',
|
|
});
|
|
assert.equal(both.reason, 'both-unhealthy');
|
|
assert.equal(both.switchTo, null);
|
|
|
|
const unknown = nextFailoverDecision({
|
|
now: 30_000,
|
|
policy: enabled,
|
|
currentRole: 'primary',
|
|
primaryHealth: 'unhealthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'unknown',
|
|
memory: { primaryFailedSince: 0, primaryRecoveredSince: null, quietSince: null },
|
|
});
|
|
assert.equal(unknown.reason, 'activity-unknown');
|
|
assert.equal(unknown.switchTo, null);
|
|
});
|
|
|
|
test('failback waits for recovery, hold and quarantine deadlines', () => {
|
|
let decision = nextFailoverDecision({
|
|
now: 10_000,
|
|
policy: enabled,
|
|
currentRole: 'reserve',
|
|
primaryHealth: 'healthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'quiet',
|
|
holdUntil: 50_000,
|
|
primaryQuarantineUntil: 70_000,
|
|
});
|
|
assert.equal(decision.nextDecisionAt, 70_000);
|
|
decision = nextFailoverDecision({
|
|
now: 70_000,
|
|
policy: enabled,
|
|
currentRole: 'reserve',
|
|
primaryHealth: 'healthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'quiet',
|
|
holdUntil: 50_000,
|
|
primaryQuarantineUntil: 70_000,
|
|
memory: decision.memory,
|
|
});
|
|
assert.equal(decision.switchTo, null);
|
|
assert.equal(decision.nextDecisionAt, 130_000);
|
|
decision = nextFailoverDecision({
|
|
now: 130_000,
|
|
policy: enabled,
|
|
currentRole: 'reserve',
|
|
primaryHealth: 'healthy',
|
|
reserveHealth: 'healthy',
|
|
activity: 'quiet',
|
|
holdUntil: 50_000,
|
|
primaryQuarantineUntil: 70_000,
|
|
memory: { ...decision.memory, quietSince: 125_000 },
|
|
});
|
|
assert.equal(decision.switchTo, 'primary');
|
|
|
|
const deadReserve = nextFailoverDecision({
|
|
now: 70_000,
|
|
policy: enabled,
|
|
currentRole: 'reserve',
|
|
primaryHealth: 'healthy',
|
|
reserveHealth: 'unhealthy',
|
|
activity: 'quiet',
|
|
holdUntil: 500_000,
|
|
primaryQuarantineUntil: 700_000,
|
|
memory: { primaryFailedSince: null, primaryRecoveredSince: 10_000, quietSince: 65_000 },
|
|
});
|
|
assert.equal(deadReserve.switchTo, null);
|
|
assert.equal(deadReserve.nextDecisionAt, 700_000);
|
|
});
|
|
|
|
function serviceState(policy = enabled) {
|
|
return {
|
|
revision: 1,
|
|
failoverPolicy: policy,
|
|
failoverRuntimeState: { lastSwitchAt: null, holdUntil: null, primaryQuarantineUntil: null, failoverHistory: [], reasonCode: null },
|
|
appliedFailoverPolicy: {
|
|
primary: { profileId: 'profile-1', serverId: 'server-1' },
|
|
reserve: { profileId: 'profile-2', serverId: 'server-2' },
|
|
primaryConfigFingerprint: 'a'.repeat(64),
|
|
reserveConfigFingerprint: 'b'.repeat(64),
|
|
},
|
|
appliedProfileId: 'profile-1',
|
|
appliedServerId: 'server-1',
|
|
profiles: [
|
|
{ id: 'profile-1', servers: [{ id: 'server-1', label: 'Primary' }] },
|
|
{ id: 'profile-2', servers: [{ id: 'server-2', label: 'Reserve' }] },
|
|
],
|
|
diagnostics: { customServices: [] },
|
|
};
|
|
}
|
|
|
|
function harness(initial = serviceState(), overrides = {}) {
|
|
let state = structuredClone(initial);
|
|
const calls = [];
|
|
const dependencies = {
|
|
state: {
|
|
read: () => state,
|
|
update: (mutator) => {
|
|
state = { ...mutator(state), revision: state.revision + 1 };
|
|
calls.push('state:update');
|
|
return state;
|
|
},
|
|
},
|
|
runtime: { isRunning: async () => true },
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async () => ({ vpn: { sites: [{ status: 'available' }] } }),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => { calls.push(`select:${role}`); return { role }; },
|
|
setFailoverActivityEnabled: async (value) => { calls.push(`activity:${value}`); },
|
|
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
|
},
|
|
buildCandidate: () => ({ config: {}, applied: initial.appliedFailoverPolicy }),
|
|
serialize: (operation) => operation(),
|
|
scheduler: {
|
|
setTimeout: () => ({ unref() {} }),
|
|
clearTimeout: () => {},
|
|
},
|
|
...overrides,
|
|
};
|
|
return { service: createFailoverService(dependencies), calls, read: () => state };
|
|
}
|
|
|
|
test('disabled failover performs one restart-safe collector cleanup and no steady-state work', async () => {
|
|
const state = serviceState(normalizeFailoverPolicy(null));
|
|
state.appliedFailoverPolicy = null;
|
|
const testHarness = harness(state);
|
|
await testHarness.service.reconcile();
|
|
assert.deepEqual(testHarness.calls, ['activity:false']);
|
|
await testHarness.service.reconcile();
|
|
assert.deepEqual(testHarness.calls, ['activity:false']);
|
|
assert.equal(testHarness.service.snapshot().activation, 'inactive');
|
|
});
|
|
|
|
test('disabled failover reports an already loaded dual config without managing it', async () => {
|
|
const testHarness = harness(serviceState(normalizeFailoverPolicy(null)));
|
|
await testHarness.service.reconcile();
|
|
assert.deepEqual(testHarness.calls, ['activity:false']);
|
|
assert.equal(testHarness.service.snapshot().activation, 'passive-loaded');
|
|
assert.equal(testHarness.service.snapshot().currentRole, 'primary');
|
|
});
|
|
|
|
test('runtime rollback restores the role from applied truth even while disabled', async () => {
|
|
const state = serviceState(normalizeFailoverPolicy(null));
|
|
state.appliedProfileId = state.appliedFailoverPolicy.reserve.profileId;
|
|
state.appliedServerId = state.appliedFailoverPolicy.reserve.serverId;
|
|
const testHarness = harness(state);
|
|
await testHarness.service.restoreAppliedActivation(state);
|
|
assert.deepEqual(testHarness.calls, ['select:reserve']);
|
|
});
|
|
|
|
test('an in-flight observation is discarded after failover is disabled', async () => {
|
|
const pending = [];
|
|
const events = [];
|
|
const testHarness = harness(serviceState(), {
|
|
onEvent: (event) => events.push(event),
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: (role) => new Promise((resolve) => pending.push({ role, resolve })),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; },
|
|
setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); },
|
|
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
|
},
|
|
});
|
|
await testHarness.service.reconcile();
|
|
const round = testHarness.service.runRound();
|
|
await new Promise((resolve) => setImmediate(resolve));
|
|
assert.equal(testHarness.service.snapshot().reason, 'checking-channels');
|
|
await testHarness.service.save({ ...enabled, enabled: false });
|
|
for (const probe of pending) probe.resolve({ vpn: { sites: [{ status: probe.role === 'reserve' ? 'available' : 'unavailable' }] } });
|
|
await round;
|
|
assert.equal(testHarness.read().failoverPolicy.enabled, false);
|
|
assert.equal(testHarness.calls.some((call) => call.startsWith('select:reserve')), false);
|
|
assert.deepEqual(testHarness.calls.filter((call) => call.startsWith('activity:')), ['activity:true', 'activity:true', 'activity:false']);
|
|
assert.deepEqual(events.filter(({ type }) => /_(?:unavailable|recovered)$/.test(type)), []);
|
|
});
|
|
|
|
test('one channel probe failure does not erase the other channel health', async () => {
|
|
const testHarness = harness(serviceState(), {
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async (role) => {
|
|
if (role === 'primary') throw new Error('primary probe transport failed');
|
|
return { vpn: { sites: [{ status: 'available' }] } };
|
|
},
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => ({ role }),
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: { state: 'active', observedAt: new Date().toISOString() } }),
|
|
},
|
|
});
|
|
await testHarness.service.checkNow();
|
|
assert.equal(testHarness.service.snapshot().primary.health, 'unknown');
|
|
assert.equal(testHarness.service.snapshot().reserve.health, 'healthy');
|
|
assert.notEqual(testHarness.service.snapshot().reason, 'checking-channels');
|
|
});
|
|
|
|
test('manual check exposes both in-flight channel roles and clears them after completion', async () => {
|
|
const pending = [];
|
|
const testHarness = harness(serviceState(), {
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: (role) => new Promise((resolve) => pending.push({ role, resolve })),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => ({ role }),
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
|
},
|
|
});
|
|
const check = testHarness.service.checkNow();
|
|
await new Promise(setImmediate);
|
|
assert.equal(testHarness.service.snapshot().reason, 'checking-channels');
|
|
for (const probe of pending) probe.resolve({ vpn: { sites: [{ status: 'available' }] } });
|
|
await check;
|
|
assert.notEqual(testHarness.service.snapshot().reason, 'checking-channels');
|
|
});
|
|
|
|
test('monitoring wakes at an earlier decision deadline instead of waiting a full interval', async () => {
|
|
let clock = 0;
|
|
const delays = [];
|
|
const testHarness = harness(serviceState(), {
|
|
now: () => new Date(clock),
|
|
scheduler: {
|
|
setTimeout: (_callback, delay) => { delays.push(delay); return { unref() {} }; },
|
|
clearTimeout: () => {},
|
|
},
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => ({ role }),
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date(clock).toISOString() } }),
|
|
},
|
|
});
|
|
await testHarness.service.reconcile();
|
|
await testHarness.service.runRound();
|
|
clock = 30_000;
|
|
await testHarness.service.runRound();
|
|
assert.equal(delays.at(-1), enabled.trafficGuard.quietWindowMs);
|
|
});
|
|
|
|
test('a late activity response cannot switch after failover is disabled', async () => {
|
|
let resolveActivity;
|
|
const activity = new Promise((resolve) => { resolveActivity = resolve; });
|
|
const testHarness = harness(serviceState(), {
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; },
|
|
setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); },
|
|
readFailoverActivity: async () => activity,
|
|
},
|
|
});
|
|
await testHarness.service.reconcile();
|
|
const round = testHarness.service.runRound();
|
|
await new Promise(setImmediate);
|
|
await testHarness.service.save({ ...enabled, enabled: false });
|
|
resolveActivity({ activity: { state: 'quiet', observedAt: new Date().toISOString() } });
|
|
await round;
|
|
assert.equal(testHarness.calls.some((value) => value.startsWith('select:')), false);
|
|
});
|
|
|
|
test('automatic switch revalidates activity immediately before selector mutation', async () => {
|
|
let clock = 0;
|
|
let activityReads = 0;
|
|
const testHarness = harness(serviceState(), {
|
|
now: () => new Date(clock),
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; },
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({
|
|
activity: {
|
|
state: ++activityReads >= 4 ? 'active' : 'quiet',
|
|
observedAt: new Date(clock).toISOString(),
|
|
},
|
|
}),
|
|
},
|
|
});
|
|
await testHarness.service.reconcile();
|
|
await testHarness.service.runRound();
|
|
clock = 30_000;
|
|
await testHarness.service.runRound();
|
|
clock = 35_000;
|
|
await testHarness.service.runRound();
|
|
assert.equal(testHarness.calls.some((value) => value.startsWith('select:')), false);
|
|
assert.equal(testHarness.service.snapshot().reason, 'revalidation-required');
|
|
});
|
|
|
|
test('selector read-back failure rolls an uncertain switch back to the canonical role', async () => {
|
|
const testHarness = harness(serviceState(), {
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async () => ({}),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => {
|
|
testHarness.calls.push(`select:${role}`);
|
|
if (role === 'reserve') throw new Error('read-back failed');
|
|
return { role };
|
|
},
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: null }),
|
|
},
|
|
});
|
|
await assert.rejects(testHarness.service.manualSwitch('reserve'), /read-back failed/);
|
|
assert.deepEqual(testHarness.calls, ['select:reserve', 'select:primary']);
|
|
assert.equal(testHarness.read().appliedServerId, 'server-1');
|
|
});
|
|
|
|
test('failed collector disable is retried because local state remains unknown', async () => {
|
|
let attempts = 0;
|
|
const testHarness = harness(serviceState(normalizeFailoverPolicy(null)), {
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async () => ({}),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async () => ({}),
|
|
setFailoverActivityEnabled: async () => {
|
|
attempts += 1;
|
|
if (attempts === 1) throw new Error('dataplane unavailable');
|
|
},
|
|
readFailoverActivity: async () => ({ activity: null }),
|
|
},
|
|
});
|
|
await assert.rejects(testHarness.service.reconcile(), /dataplane unavailable/);
|
|
await testHarness.service.reconcile();
|
|
assert.equal(attempts, 2);
|
|
});
|
|
|
|
test('each monitoring round repairs selector and collector state after a dataplane restart', async () => {
|
|
const state = serviceState();
|
|
state.appliedProfileId = 'profile-2';
|
|
state.appliedServerId = 'server-2';
|
|
let selected = 'reserve';
|
|
let enables = 0;
|
|
const testHarness = harness(state, {
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async () => ({ vpn: { sites: [{ status: 'available' }] } }),
|
|
readFailoverSelector: async () => ({ role: selected }),
|
|
selectFailoverRole: async (role) => { selected = role; return { role }; },
|
|
setFailoverActivityEnabled: async (value) => { if (value) enables += 1; },
|
|
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
|
},
|
|
});
|
|
await testHarness.service.reconcile();
|
|
selected = 'primary';
|
|
await testHarness.service.runRound();
|
|
assert.equal(selected, 'reserve');
|
|
assert.equal(enables, 2);
|
|
});
|
|
|
|
test('selector acknowledgement precedes state commit and a failed commit rolls selector back', async () => {
|
|
const state = serviceState();
|
|
const calls = [];
|
|
const testHarness = harness(state, {
|
|
state: {
|
|
read: () => state,
|
|
update: (mutator) => {
|
|
const candidate = mutator(state);
|
|
calls.push('state:update');
|
|
if (candidate.appliedServerId === 'server-2') throw new Error('write failed');
|
|
Object.assign(state, candidate);
|
|
return state;
|
|
},
|
|
},
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async () => ({}),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => { calls.push(`select:${role}`); return { role }; },
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: null }),
|
|
},
|
|
});
|
|
await assert.rejects(testHarness.service.manualSwitch('reserve'), /write failed/);
|
|
assert.deepEqual(calls, ['select:reserve', 'state:update', 'select:primary']);
|
|
assert.equal(state.appliedServerId, 'server-1');
|
|
});
|
|
|
|
test('manual switch preserves automation and starts the configured reserve hold', async () => {
|
|
const now = new Date('2026-08-27T12:00:00.000Z');
|
|
for (const paused of [false, true]) {
|
|
const state = serviceState(normalizeFailoverPolicy({
|
|
...enabled,
|
|
paused,
|
|
minimumReserveMs: 300_000,
|
|
}));
|
|
const testHarness = harness(state, { now: () => now });
|
|
await testHarness.service.manualSwitch('reserve');
|
|
assert.equal(testHarness.read().appliedServerId, 'server-2');
|
|
assert.equal(testHarness.read().failoverPolicy.enabled, true);
|
|
assert.equal(testHarness.read().failoverPolicy.paused, paused);
|
|
assert.equal(testHarness.read().failoverRuntimeState.holdUntil, new Date(now.getTime() + 300_000).toISOString());
|
|
assert.equal(testHarness.calls.filter((call) => call === 'state:update').length, 1);
|
|
}
|
|
});
|
|
|
|
test('post-commit monitoring failure does not report a committed pause as failed', async () => {
|
|
const warnings = [];
|
|
const testHarness = harness(serviceState(), {
|
|
runtime: { isRunning: async () => { throw new Error('runtime unavailable'); } },
|
|
onWarning: (error) => warnings.push(error.message),
|
|
});
|
|
await testHarness.service.pause(true);
|
|
assert.equal(testHarness.read().failoverPolicy.paused, true);
|
|
assert.equal(testHarness.service.snapshot().status, 'error');
|
|
assert.deepEqual(warnings, ['runtime unavailable']);
|
|
});
|
|
|
|
test('pause and resume discard stale failure-window evidence', async () => {
|
|
let clock = 1_000;
|
|
const testHarness = harness(serviceState(), {
|
|
now: () => new Date(clock),
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async (role) => ({
|
|
vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] },
|
|
}),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; },
|
|
setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); },
|
|
readFailoverActivity: async () => ({
|
|
activity: { state: 'quiet', observedAt: new Date(clock).toISOString(), quietSince: new Date(clock - 10_000).toISOString() },
|
|
}),
|
|
},
|
|
});
|
|
await testHarness.service.runRound();
|
|
await testHarness.service.pause(true);
|
|
clock = 3_600_000;
|
|
await testHarness.service.pause(false);
|
|
await testHarness.service.runRound();
|
|
assert.equal(testHarness.calls.includes('select:reserve'), false);
|
|
assert.equal(testHarness.service.snapshot().reason, 'failure-window');
|
|
});
|
|
|
|
test('an unconfirmed selector rollback durably pauses automation', async () => {
|
|
const testHarness = harness(serviceState(), {
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async () => ({}),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async () => { throw new Error('selector unavailable'); },
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: null }),
|
|
},
|
|
});
|
|
await assert.rejects(testHarness.service.manualSwitch('reserve'), AggregateError);
|
|
assert.equal(testHarness.read().failoverPolicy.paused, true);
|
|
assert.equal(testHarness.read().failoverRuntimeState.reasonCode, 'selector-unknown');
|
|
assert.equal(testHarness.service.snapshot().reason, 'selector-unknown');
|
|
assert.equal(testHarness.service.snapshot().currentRole, 'other');
|
|
});
|
|
|
|
test('successful selector reconciliation clears an unknown-role latch', async () => {
|
|
const state = serviceState();
|
|
state.failoverRuntimeState.reasonCode = 'selector-unknown';
|
|
const testHarness = harness(state);
|
|
await testHarness.service.reconcile();
|
|
assert.equal(testHarness.read().failoverRuntimeState.reasonCode, null);
|
|
assert.equal(testHarness.service.snapshot().currentRole, 'primary');
|
|
});
|
|
|
|
test('disabling on reserve preserves it as the next ordinary single-channel target', async () => {
|
|
const state = serviceState();
|
|
state.appliedProfileId = 'profile-2';
|
|
state.appliedServerId = 'server-2';
|
|
const testHarness = harness(state);
|
|
await testHarness.service.save({ ...enabled, enabled: false });
|
|
assert.equal(testHarness.read().desiredProfileId, 'profile-2');
|
|
assert.equal(testHarness.read().profiles[1].desiredServerId, 'server-2');
|
|
assert.equal(testHarness.service.snapshot().activation, 'passive-loaded');
|
|
});
|
|
|
|
test('background health transitions cover both channels without duplicate observations', async () => {
|
|
let clock = 0;
|
|
const health = { primary: 'unavailable', reserve: 'available' };
|
|
const events = [];
|
|
const testHarness = harness(serviceState(), {
|
|
now: () => new Date(clock),
|
|
onEvent: (event) => events.push(event),
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: health[role] }] } }),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => ({ role }),
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date(clock).toISOString() } }),
|
|
},
|
|
});
|
|
const healthEvents = () => events.filter(({ type }) => /_(?:unavailable|recovered)$/.test(type));
|
|
const round = async () => {
|
|
await testHarness.service.runRound();
|
|
clock += 1_000;
|
|
};
|
|
|
|
await testHarness.service.reconcile();
|
|
await round();
|
|
await round();
|
|
health.reserve = 'unavailable';
|
|
await round();
|
|
await round();
|
|
health.primary = 'available';
|
|
await round();
|
|
health.primary = 'unavailable';
|
|
await round();
|
|
health.reserve = 'available';
|
|
await round();
|
|
health.reserve = 'unavailable';
|
|
await round();
|
|
|
|
assert.deepEqual(healthEvents().map(({ type, severity, data }) => ({ type, severity, role: data.role })), [
|
|
{ type: 'failover.primary_unavailable', severity: 'warning', role: 'primary' },
|
|
{ type: 'failover.reserve_unavailable', severity: 'warning', role: 'reserve' },
|
|
{ type: 'failover.primary_recovered', severity: 'info', role: 'primary' },
|
|
{ type: 'failover.primary_unavailable', severity: 'warning', role: 'primary' },
|
|
{ type: 'failover.reserve_recovered', severity: 'info', role: 'reserve' },
|
|
{ type: 'failover.reserve_unavailable', severity: 'warning', role: 'reserve' },
|
|
]);
|
|
assert.ok(healthEvents().every(({ dedupeKey }) => dedupeKey === null));
|
|
assert.equal(events.some(({ type }) => type === 'failover.recovered'), false);
|
|
assert.equal(events.filter(({ type }) => type === 'failover.both_unhealthy').length, 3);
|
|
});
|
|
|
|
test('activity read failure does not hide completed channel health transitions', async () => {
|
|
const events = [];
|
|
const testHarness = harness(serviceState(), {
|
|
onEvent: (event) => events.push(event),
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async (role) => ({
|
|
vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] },
|
|
}),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => ({ role }),
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => { throw new Error('activity unavailable'); },
|
|
},
|
|
});
|
|
|
|
await testHarness.service.reconcile();
|
|
await assert.rejects(testHarness.service.runRound(), /activity unavailable/);
|
|
assert.deepEqual(events.map(({ type }) => type), ['failover.primary_unavailable']);
|
|
assert.equal(events.some(({ type }) => ['failover.waiting_for_idle', 'failover.both_unhealthy', 'failover.switched', 'failover.switch_failed'].includes(type)), false);
|
|
});
|
|
|
|
test('manual check does not strand monitoring behind a stale background round', async () => {
|
|
const pending = [];
|
|
const scheduled = [];
|
|
let holdBackground = true;
|
|
let probeCalls = 0;
|
|
const testHarness = harness(serviceState(), {
|
|
scheduler: {
|
|
setTimeout: (callback) => {
|
|
scheduled.push(callback);
|
|
return { unref() {} };
|
|
},
|
|
clearTimeout: () => {},
|
|
},
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: (role) => {
|
|
probeCalls += 1;
|
|
if (holdBackground) {
|
|
return new Promise((resolve) => pending.push({ role, resolve }));
|
|
}
|
|
return Promise.resolve({ vpn: { sites: [{ status: 'available' }] } });
|
|
},
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => ({ role }),
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
|
},
|
|
});
|
|
|
|
await testHarness.service.reconcile();
|
|
const oldRound = testHarness.service.runRound();
|
|
await new Promise(setImmediate);
|
|
assert.equal(pending.length, 4);
|
|
holdBackground = false;
|
|
await testHarness.service.checkNow();
|
|
assert.equal(probeCalls, 8);
|
|
scheduled.at(-1)();
|
|
for (const probe of pending) {
|
|
probe.resolve({ vpn: { sites: [{ status: 'available' }] } });
|
|
}
|
|
await oldRound;
|
|
await new Promise(setImmediate);
|
|
await new Promise(setImmediate);
|
|
assert.equal(probeCalls, 12);
|
|
});
|
|
|
|
test('manual and background checks share channel transition memory', async () => {
|
|
const health = { primary: 'available', reserve: 'available' };
|
|
const events = [];
|
|
const testHarness = harness(serviceState(), {
|
|
onEvent: (event) => events.push(event),
|
|
dataplane: {
|
|
checkConfig: async () => ({}),
|
|
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: health[role] }] } }),
|
|
readFailoverSelector: async () => ({ role: 'primary' }),
|
|
selectFailoverRole: async (role) => ({ role }),
|
|
setFailoverActivityEnabled: async () => {},
|
|
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
|
},
|
|
});
|
|
const healthTypes = () => events
|
|
.filter(({ type }) => /_(?:unavailable|recovered)$/.test(type))
|
|
.map(({ type }) => type);
|
|
|
|
await testHarness.service.reconcile();
|
|
await testHarness.service.checkNow();
|
|
assert.deepEqual(healthTypes(), []);
|
|
health.primary = 'unavailable';
|
|
await testHarness.service.checkNow();
|
|
assert.deepEqual(healthTypes(), ['failover.primary_unavailable']);
|
|
await testHarness.service.runRound();
|
|
assert.deepEqual(healthTypes(), ['failover.primary_unavailable']);
|
|
health.primary = 'available';
|
|
await testHarness.service.checkNow();
|
|
health.primary = 'unavailable';
|
|
await testHarness.service.runRound();
|
|
health.reserve = 'unavailable';
|
|
await testHarness.service.checkNow();
|
|
await testHarness.service.runRound();
|
|
health.reserve = 'available';
|
|
await testHarness.service.checkNow();
|
|
health.reserve = 'unavailable';
|
|
await testHarness.service.runRound();
|
|
|
|
assert.deepEqual(healthTypes(), [
|
|
'failover.primary_unavailable',
|
|
'failover.primary_recovered',
|
|
'failover.primary_unavailable',
|
|
'failover.reserve_unavailable',
|
|
'failover.reserve_recovered',
|
|
'failover.reserve_unavailable',
|
|
]);
|
|
});
|
|
|
|
test('pending target edits keep selector commits bound to the loaded channel pair', async () => {
|
|
const state = serviceState();
|
|
state.failoverPolicy = normalizeFailoverPolicy({
|
|
...state.failoverPolicy,
|
|
reserve: { profileId: 'profile-3', serverId: 'server-3' },
|
|
});
|
|
state.profiles.push({ id: 'profile-3', servers: [{ id: 'server-3', label: 'Desired later' }] });
|
|
const testHarness = harness(state, {
|
|
buildCandidate: () => ({
|
|
config: {},
|
|
applied: {
|
|
...state.appliedFailoverPolicy,
|
|
reserve: { profileId: 'profile-3', serverId: 'server-3' },
|
|
reserveConfigFingerprint: 'c'.repeat(64),
|
|
},
|
|
}),
|
|
});
|
|
|
|
await testHarness.service.manualSwitch('reserve');
|
|
assert.equal(testHarness.read().appliedProfileId, 'profile-2');
|
|
assert.equal(testHarness.read().appliedServerId, 'server-2');
|
|
assert.equal(testHarness.service.snapshot().activation, 'pending');
|
|
assert.deepEqual(testHarness.service.snapshot().reserve.target, { profileId: 'profile-2', serverId: 'server-2' });
|
|
});
|