Update Harbor client and gateway integration workflows
This commit is contained in:
@@ -0,0 +1,635 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
DEFAULT_FAILOVER_POLICY,
|
||||
isFailoverConfigured,
|
||||
nextFailoverDecision,
|
||||
normalizeFailoverPolicy,
|
||||
} from '../../dist/shared/failover.js';
|
||||
import { createFailoverService } from '../../dist/server/features/failover/failoverService.js';
|
||||
|
||||
const enabled = normalizeFailoverPolicy({
|
||||
...DEFAULT_FAILOVER_POLICY,
|
||||
enabled: true,
|
||||
primary: { profileId: 'profile-1', serverId: 'server-1' },
|
||||
reserve: { profileId: 'profile-2', serverId: 'server-2' },
|
||||
intervalMs: 15_000,
|
||||
failureWindowMs: 30_000,
|
||||
recoveryWindowMs: 60_000,
|
||||
trafficGuard: { enabled: true, thresholdBytesPerSecond: 1024, quietWindowMs: 5_000 },
|
||||
minimumReserveMs: 60_000,
|
||||
});
|
||||
|
||||
test('failover policy defaults to strict disabled zero-work state', () => {
|
||||
const policy = normalizeFailoverPolicy(null);
|
||||
const decision = nextFailoverDecision({
|
||||
now: 0,
|
||||
policy,
|
||||
currentRole: 'primary',
|
||||
primaryHealth: 'unhealthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'quiet',
|
||||
});
|
||||
assert.equal(policy.enabled, false);
|
||||
assert.equal(decision.status, 'idle');
|
||||
assert.equal(decision.switchTo, null);
|
||||
});
|
||||
|
||||
test('failover save rejects policy values that would otherwise be silently clamped', async () => {
|
||||
const testHarness = harness(serviceState());
|
||||
await assert.rejects(testHarness.service.save({ ...enabled, intervalMs: -1 }), { code: 'REQUEST_INVALID' });
|
||||
});
|
||||
|
||||
test('strict failover validation accepts semantically identical reordered JSON keys', () => {
|
||||
const reordered = {
|
||||
flapProtection: enabled.flapProtection,
|
||||
minimumReserveMs: enabled.minimumReserveMs,
|
||||
trafficGuard: enabled.trafficGuard,
|
||||
recoveryWindowMs: enabled.recoveryWindowMs,
|
||||
failureWindowMs: enabled.failureWindowMs,
|
||||
intervalMs: enabled.intervalMs,
|
||||
checks: enabled.checks,
|
||||
reserve: enabled.reserve,
|
||||
primary: enabled.primary,
|
||||
paused: enabled.paused,
|
||||
enabled: enabled.enabled,
|
||||
version: enabled.version,
|
||||
};
|
||||
assert.deepEqual(normalizeFailoverPolicy(reordered, { strict: true }), enabled);
|
||||
});
|
||||
|
||||
test('enabled failover requires two distinct existing targets', async () => {
|
||||
const sameTarget = normalizeFailoverPolicy({ ...enabled, reserve: enabled.primary });
|
||||
assert.equal(isFailoverConfigured(sameTarget), false);
|
||||
await assert.rejects(harness(serviceState()).service.save(sameTarget), { code: 'REQUEST_INVALID' });
|
||||
|
||||
const state = serviceState({ ...enabled, paused: true });
|
||||
state.profiles[1].servers = [];
|
||||
const testHarness = harness(state);
|
||||
await assert.rejects(testHarness.service.pause(false), { code: 'REQUEST_INVALID' });
|
||||
assert.equal(testHarness.read().failoverPolicy.paused, true);
|
||||
});
|
||||
|
||||
test('local Gateway activity uses the existing domain traffic collector only while enabled', () => {
|
||||
const source = fs.readFileSync(new URL('../../src/server/index.ts', import.meta.url), 'utf8');
|
||||
assert.match(source, /const localFailoverTraffic = !remoteDataplane && settings\.appMode === 'gateway'[\s\S]*createDomainTrafficService/);
|
||||
assert.match(source, /setLocalFailoverActivityEnabled[\s\S]*enableActivity\(\)[\s\S]*setInterval\(refresh, 2_000\)/);
|
||||
assert.match(source, /if \(!enabled\)[\s\S]*clearInterval\(localFailoverTrafficTimer\)[\s\S]*disableActivity\(\)/);
|
||||
assert.doesNotMatch(source, /setFailoverActivityEnabled: async \(\) => \(\{\}\)/);
|
||||
});
|
||||
|
||||
test('failure window and traffic guard delay reserve switch until continuous quiet', () => {
|
||||
let decision = nextFailoverDecision({
|
||||
now: 0,
|
||||
policy: enabled,
|
||||
currentRole: 'primary',
|
||||
primaryHealth: 'unhealthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'active',
|
||||
});
|
||||
decision = nextFailoverDecision({
|
||||
now: 30_000,
|
||||
policy: enabled,
|
||||
currentRole: 'primary',
|
||||
primaryHealth: 'unhealthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'active',
|
||||
memory: decision.memory,
|
||||
});
|
||||
assert.equal(decision.status, 'waiting-for-idle');
|
||||
decision = nextFailoverDecision({
|
||||
now: 31_000,
|
||||
policy: enabled,
|
||||
currentRole: 'primary',
|
||||
primaryHealth: 'unhealthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'quiet',
|
||||
memory: decision.memory,
|
||||
});
|
||||
assert.equal(decision.switchTo, null);
|
||||
decision = nextFailoverDecision({
|
||||
now: 36_000,
|
||||
policy: enabled,
|
||||
currentRole: 'primary',
|
||||
primaryHealth: 'unhealthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'quiet',
|
||||
memory: decision.memory,
|
||||
});
|
||||
assert.equal(decision.switchTo, 'reserve');
|
||||
});
|
||||
|
||||
test('both unhealthy and unknown activity never cause a switch', () => {
|
||||
const both = nextFailoverDecision({
|
||||
now: 30_000,
|
||||
policy: enabled,
|
||||
currentRole: 'primary',
|
||||
primaryHealth: 'unhealthy',
|
||||
reserveHealth: 'unhealthy',
|
||||
activity: 'quiet',
|
||||
});
|
||||
assert.equal(both.reason, 'both-unhealthy');
|
||||
assert.equal(both.switchTo, null);
|
||||
|
||||
const unknown = nextFailoverDecision({
|
||||
now: 30_000,
|
||||
policy: enabled,
|
||||
currentRole: 'primary',
|
||||
primaryHealth: 'unhealthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'unknown',
|
||||
memory: { primaryFailedSince: 0, primaryRecoveredSince: null, quietSince: null },
|
||||
});
|
||||
assert.equal(unknown.reason, 'activity-unknown');
|
||||
assert.equal(unknown.switchTo, null);
|
||||
});
|
||||
|
||||
test('failback waits for recovery, hold and quarantine deadlines', () => {
|
||||
let decision = nextFailoverDecision({
|
||||
now: 10_000,
|
||||
policy: enabled,
|
||||
currentRole: 'reserve',
|
||||
primaryHealth: 'healthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'quiet',
|
||||
holdUntil: 50_000,
|
||||
primaryQuarantineUntil: 70_000,
|
||||
});
|
||||
assert.equal(decision.nextDecisionAt, 70_000);
|
||||
decision = nextFailoverDecision({
|
||||
now: 70_000,
|
||||
policy: enabled,
|
||||
currentRole: 'reserve',
|
||||
primaryHealth: 'healthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'quiet',
|
||||
holdUntil: 50_000,
|
||||
primaryQuarantineUntil: 70_000,
|
||||
memory: decision.memory,
|
||||
});
|
||||
assert.equal(decision.switchTo, null);
|
||||
assert.equal(decision.nextDecisionAt, 130_000);
|
||||
decision = nextFailoverDecision({
|
||||
now: 130_000,
|
||||
policy: enabled,
|
||||
currentRole: 'reserve',
|
||||
primaryHealth: 'healthy',
|
||||
reserveHealth: 'healthy',
|
||||
activity: 'quiet',
|
||||
holdUntil: 50_000,
|
||||
primaryQuarantineUntil: 70_000,
|
||||
memory: { ...decision.memory, quietSince: 125_000 },
|
||||
});
|
||||
assert.equal(decision.switchTo, 'primary');
|
||||
|
||||
const deadReserve = nextFailoverDecision({
|
||||
now: 70_000,
|
||||
policy: enabled,
|
||||
currentRole: 'reserve',
|
||||
primaryHealth: 'healthy',
|
||||
reserveHealth: 'unhealthy',
|
||||
activity: 'quiet',
|
||||
holdUntil: 500_000,
|
||||
primaryQuarantineUntil: 700_000,
|
||||
memory: { primaryFailedSince: null, primaryRecoveredSince: 10_000, quietSince: 65_000 },
|
||||
});
|
||||
assert.equal(deadReserve.switchTo, null);
|
||||
assert.equal(deadReserve.nextDecisionAt, 700_000);
|
||||
});
|
||||
|
||||
function serviceState(policy = enabled) {
|
||||
return {
|
||||
revision: 1,
|
||||
failoverPolicy: policy,
|
||||
failoverRuntimeState: { lastSwitchAt: null, holdUntil: null, primaryQuarantineUntil: null, failoverHistory: [], reasonCode: null },
|
||||
appliedFailoverPolicy: {
|
||||
primary: { profileId: 'profile-1', serverId: 'server-1' },
|
||||
reserve: { profileId: 'profile-2', serverId: 'server-2' },
|
||||
primaryConfigFingerprint: 'a'.repeat(64),
|
||||
reserveConfigFingerprint: 'b'.repeat(64),
|
||||
},
|
||||
appliedProfileId: 'profile-1',
|
||||
appliedServerId: 'server-1',
|
||||
profiles: [
|
||||
{ id: 'profile-1', servers: [{ id: 'server-1', label: 'Primary' }] },
|
||||
{ id: 'profile-2', servers: [{ id: 'server-2', label: 'Reserve' }] },
|
||||
],
|
||||
diagnostics: { customServices: [] },
|
||||
};
|
||||
}
|
||||
|
||||
function harness(initial = serviceState(), overrides = {}) {
|
||||
let state = structuredClone(initial);
|
||||
const calls = [];
|
||||
const dependencies = {
|
||||
state: {
|
||||
read: () => state,
|
||||
update: (mutator) => {
|
||||
state = { ...mutator(state), revision: state.revision + 1 };
|
||||
calls.push('state:update');
|
||||
return state;
|
||||
},
|
||||
},
|
||||
runtime: { isRunning: async () => true },
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async () => ({ vpn: { sites: [{ status: 'available' }] } }),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => { calls.push(`select:${role}`); return { role }; },
|
||||
setFailoverActivityEnabled: async (value) => { calls.push(`activity:${value}`); },
|
||||
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
||||
},
|
||||
buildCandidate: () => ({ config: {}, applied: initial.appliedFailoverPolicy }),
|
||||
serialize: (operation) => operation(),
|
||||
scheduler: {
|
||||
setTimeout: () => ({ unref() {} }),
|
||||
clearTimeout: () => {},
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
return { service: createFailoverService(dependencies), calls, read: () => state };
|
||||
}
|
||||
|
||||
test('disabled failover performs one restart-safe collector cleanup and no steady-state work', async () => {
|
||||
const state = serviceState(normalizeFailoverPolicy(null));
|
||||
state.appliedFailoverPolicy = null;
|
||||
const testHarness = harness(state);
|
||||
await testHarness.service.reconcile();
|
||||
assert.deepEqual(testHarness.calls, ['activity:false']);
|
||||
await testHarness.service.reconcile();
|
||||
assert.deepEqual(testHarness.calls, ['activity:false']);
|
||||
assert.equal(testHarness.service.snapshot().activation, 'inactive');
|
||||
});
|
||||
|
||||
test('disabled failover reports an already loaded dual config without managing it', async () => {
|
||||
const testHarness = harness(serviceState(normalizeFailoverPolicy(null)));
|
||||
await testHarness.service.reconcile();
|
||||
assert.deepEqual(testHarness.calls, ['activity:false']);
|
||||
assert.equal(testHarness.service.snapshot().activation, 'passive-loaded');
|
||||
assert.equal(testHarness.service.snapshot().currentRole, 'primary');
|
||||
});
|
||||
|
||||
test('runtime rollback restores the role from applied truth even while disabled', async () => {
|
||||
const state = serviceState(normalizeFailoverPolicy(null));
|
||||
state.appliedProfileId = state.appliedFailoverPolicy.reserve.profileId;
|
||||
state.appliedServerId = state.appliedFailoverPolicy.reserve.serverId;
|
||||
const testHarness = harness(state);
|
||||
await testHarness.service.restoreAppliedActivation(state);
|
||||
assert.deepEqual(testHarness.calls, ['select:reserve']);
|
||||
});
|
||||
|
||||
test('an in-flight observation is discarded after failover is disabled', async () => {
|
||||
const pending = [];
|
||||
const testHarness = harness(serviceState(), {
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: (role) => new Promise((resolve) => pending.push({ role, resolve })),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; },
|
||||
setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); },
|
||||
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
||||
},
|
||||
});
|
||||
await testHarness.service.reconcile();
|
||||
const round = testHarness.service.runRound();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
await testHarness.service.save({ ...enabled, enabled: false });
|
||||
for (const probe of pending) probe.resolve({ vpn: { sites: [{ status: probe.role === 'reserve' ? 'available' : 'unavailable' }] } });
|
||||
await round;
|
||||
assert.equal(testHarness.read().failoverPolicy.enabled, false);
|
||||
assert.equal(testHarness.calls.some((call) => call.startsWith('select:reserve')), false);
|
||||
assert.deepEqual(testHarness.calls.filter((call) => call.startsWith('activity:')), ['activity:true', 'activity:true', 'activity:false']);
|
||||
});
|
||||
|
||||
test('one channel probe failure does not erase the other channel health', async () => {
|
||||
const testHarness = harness(serviceState(), {
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async (role) => {
|
||||
if (role === 'primary') throw new Error('primary probe transport failed');
|
||||
return { vpn: { sites: [{ status: 'available' }] } };
|
||||
},
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => ({ role }),
|
||||
setFailoverActivityEnabled: async () => {},
|
||||
readFailoverActivity: async () => ({ activity: { state: 'active', observedAt: new Date().toISOString() } }),
|
||||
},
|
||||
});
|
||||
await testHarness.service.checkNow();
|
||||
assert.equal(testHarness.service.snapshot().primary.health, 'unknown');
|
||||
assert.equal(testHarness.service.snapshot().reserve.health, 'healthy');
|
||||
});
|
||||
|
||||
test('monitoring wakes at an earlier decision deadline instead of waiting a full interval', async () => {
|
||||
let clock = 0;
|
||||
const delays = [];
|
||||
const testHarness = harness(serviceState(), {
|
||||
now: () => new Date(clock),
|
||||
scheduler: {
|
||||
setTimeout: (_callback, delay) => { delays.push(delay); return { unref() {} }; },
|
||||
clearTimeout: () => {},
|
||||
},
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => ({ role }),
|
||||
setFailoverActivityEnabled: async () => {},
|
||||
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date(clock).toISOString() } }),
|
||||
},
|
||||
});
|
||||
await testHarness.service.reconcile();
|
||||
await testHarness.service.runRound();
|
||||
clock = 30_000;
|
||||
await testHarness.service.runRound();
|
||||
assert.equal(delays.at(-1), enabled.trafficGuard.quietWindowMs);
|
||||
});
|
||||
|
||||
test('a late activity response cannot switch after failover is disabled', async () => {
|
||||
let resolveActivity;
|
||||
const activity = new Promise((resolve) => { resolveActivity = resolve; });
|
||||
const testHarness = harness(serviceState(), {
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; },
|
||||
setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); },
|
||||
readFailoverActivity: async () => activity,
|
||||
},
|
||||
});
|
||||
await testHarness.service.reconcile();
|
||||
const round = testHarness.service.runRound();
|
||||
await new Promise(setImmediate);
|
||||
await testHarness.service.save({ ...enabled, enabled: false });
|
||||
resolveActivity({ activity: { state: 'quiet', observedAt: new Date().toISOString() } });
|
||||
await round;
|
||||
assert.equal(testHarness.calls.some((value) => value.startsWith('select:')), false);
|
||||
});
|
||||
|
||||
test('automatic switch revalidates activity immediately before selector mutation', async () => {
|
||||
let clock = 0;
|
||||
let activityReads = 0;
|
||||
const testHarness = harness(serviceState(), {
|
||||
now: () => new Date(clock),
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] } }),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; },
|
||||
setFailoverActivityEnabled: async () => {},
|
||||
readFailoverActivity: async () => ({
|
||||
activity: {
|
||||
state: ++activityReads >= 4 ? 'active' : 'quiet',
|
||||
observedAt: new Date(clock).toISOString(),
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
await testHarness.service.reconcile();
|
||||
await testHarness.service.runRound();
|
||||
clock = 30_000;
|
||||
await testHarness.service.runRound();
|
||||
clock = 35_000;
|
||||
await testHarness.service.runRound();
|
||||
assert.equal(testHarness.calls.some((value) => value.startsWith('select:')), false);
|
||||
assert.equal(testHarness.service.snapshot().reason, 'revalidation-required');
|
||||
});
|
||||
|
||||
test('selector read-back failure rolls an uncertain switch back to the canonical role', async () => {
|
||||
const testHarness = harness(serviceState(), {
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async () => ({}),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => {
|
||||
testHarness.calls.push(`select:${role}`);
|
||||
if (role === 'reserve') throw new Error('read-back failed');
|
||||
return { role };
|
||||
},
|
||||
setFailoverActivityEnabled: async () => {},
|
||||
readFailoverActivity: async () => ({ activity: null }),
|
||||
},
|
||||
});
|
||||
await assert.rejects(testHarness.service.manualSwitch('reserve'), /read-back failed/);
|
||||
assert.deepEqual(testHarness.calls, ['select:reserve', 'select:primary']);
|
||||
assert.equal(testHarness.read().appliedServerId, 'server-1');
|
||||
});
|
||||
|
||||
test('failed collector disable is retried because local state remains unknown', async () => {
|
||||
let attempts = 0;
|
||||
const testHarness = harness(serviceState(normalizeFailoverPolicy(null)), {
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async () => ({}),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async () => ({}),
|
||||
setFailoverActivityEnabled: async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) throw new Error('dataplane unavailable');
|
||||
},
|
||||
readFailoverActivity: async () => ({ activity: null }),
|
||||
},
|
||||
});
|
||||
await assert.rejects(testHarness.service.reconcile(), /dataplane unavailable/);
|
||||
await testHarness.service.reconcile();
|
||||
assert.equal(attempts, 2);
|
||||
});
|
||||
|
||||
test('each monitoring round repairs selector and collector state after a dataplane restart', async () => {
|
||||
const state = serviceState();
|
||||
state.appliedProfileId = 'profile-2';
|
||||
state.appliedServerId = 'server-2';
|
||||
let selected = 'reserve';
|
||||
let enables = 0;
|
||||
const testHarness = harness(state, {
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async () => ({ vpn: { sites: [{ status: 'available' }] } }),
|
||||
readFailoverSelector: async () => ({ role: selected }),
|
||||
selectFailoverRole: async (role) => { selected = role; return { role }; },
|
||||
setFailoverActivityEnabled: async (value) => { if (value) enables += 1; },
|
||||
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
|
||||
},
|
||||
});
|
||||
await testHarness.service.reconcile();
|
||||
selected = 'primary';
|
||||
await testHarness.service.runRound();
|
||||
assert.equal(selected, 'reserve');
|
||||
assert.equal(enables, 2);
|
||||
});
|
||||
|
||||
test('selector acknowledgement precedes state commit and a failed commit rolls selector back', async () => {
|
||||
const state = serviceState();
|
||||
const calls = [];
|
||||
const testHarness = harness(state, {
|
||||
state: {
|
||||
read: () => state,
|
||||
update: (mutator) => {
|
||||
const candidate = mutator(state);
|
||||
calls.push('state:update');
|
||||
if (candidate.appliedServerId === 'server-2') throw new Error('write failed');
|
||||
Object.assign(state, candidate);
|
||||
return state;
|
||||
},
|
||||
},
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async () => ({}),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => { calls.push(`select:${role}`); return { role }; },
|
||||
setFailoverActivityEnabled: async () => {},
|
||||
readFailoverActivity: async () => ({ activity: null }),
|
||||
},
|
||||
});
|
||||
await assert.rejects(testHarness.service.manualSwitch('reserve'), /write failed/);
|
||||
assert.deepEqual(calls, ['select:reserve', 'state:update', 'select:primary']);
|
||||
assert.equal(state.appliedServerId, 'server-1');
|
||||
});
|
||||
|
||||
test('manual selector and pause publish in one canonical commit', async () => {
|
||||
const testHarness = harness(serviceState());
|
||||
await testHarness.service.manualSwitch('reserve');
|
||||
assert.equal(testHarness.read().appliedServerId, 'server-2');
|
||||
assert.equal(testHarness.read().failoverPolicy.paused, true);
|
||||
assert.equal(testHarness.calls.filter((call) => call === 'state:update').length, 1);
|
||||
});
|
||||
|
||||
test('post-commit monitoring failure does not report a committed pause as failed', async () => {
|
||||
const warnings = [];
|
||||
const testHarness = harness(serviceState(), {
|
||||
runtime: { isRunning: async () => { throw new Error('runtime unavailable'); } },
|
||||
onWarning: (error) => warnings.push(error.message),
|
||||
});
|
||||
await testHarness.service.pause(true);
|
||||
assert.equal(testHarness.read().failoverPolicy.paused, true);
|
||||
assert.equal(testHarness.service.snapshot().status, 'error');
|
||||
assert.deepEqual(warnings, ['runtime unavailable']);
|
||||
});
|
||||
|
||||
test('pause and resume discard stale failure-window evidence', async () => {
|
||||
let clock = 1_000;
|
||||
const testHarness = harness(serviceState(), {
|
||||
now: () => new Date(clock),
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async (role) => ({
|
||||
vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] },
|
||||
}),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => { testHarness.calls.push(`select:${role}`); return { role }; },
|
||||
setFailoverActivityEnabled: async (value) => { testHarness.calls.push(`activity:${value}`); },
|
||||
readFailoverActivity: async () => ({
|
||||
activity: { state: 'quiet', observedAt: new Date(clock).toISOString(), quietSince: new Date(clock - 10_000).toISOString() },
|
||||
}),
|
||||
},
|
||||
});
|
||||
await testHarness.service.runRound();
|
||||
await testHarness.service.pause(true);
|
||||
clock = 3_600_000;
|
||||
await testHarness.service.pause(false);
|
||||
await testHarness.service.runRound();
|
||||
assert.equal(testHarness.calls.includes('select:reserve'), false);
|
||||
assert.equal(testHarness.service.snapshot().reason, 'failure-window');
|
||||
});
|
||||
|
||||
test('an unconfirmed selector rollback durably pauses automation', async () => {
|
||||
const testHarness = harness(serviceState(), {
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async () => ({}),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async () => { throw new Error('selector unavailable'); },
|
||||
setFailoverActivityEnabled: async () => {},
|
||||
readFailoverActivity: async () => ({ activity: null }),
|
||||
},
|
||||
});
|
||||
await assert.rejects(testHarness.service.manualSwitch('reserve'), AggregateError);
|
||||
assert.equal(testHarness.read().failoverPolicy.paused, true);
|
||||
assert.equal(testHarness.read().failoverRuntimeState.reasonCode, 'selector-unknown');
|
||||
assert.equal(testHarness.service.snapshot().reason, 'selector-unknown');
|
||||
assert.equal(testHarness.service.snapshot().currentRole, 'other');
|
||||
});
|
||||
|
||||
test('successful selector reconciliation clears an unknown-role latch', async () => {
|
||||
const state = serviceState();
|
||||
state.failoverRuntimeState.reasonCode = 'selector-unknown';
|
||||
const testHarness = harness(state);
|
||||
await testHarness.service.reconcile();
|
||||
assert.equal(testHarness.read().failoverRuntimeState.reasonCode, null);
|
||||
assert.equal(testHarness.service.snapshot().currentRole, 'primary');
|
||||
});
|
||||
|
||||
test('disabling on reserve preserves it as the next ordinary single-channel target', async () => {
|
||||
const state = serviceState();
|
||||
state.appliedProfileId = 'profile-2';
|
||||
state.appliedServerId = 'server-2';
|
||||
const testHarness = harness(state);
|
||||
await testHarness.service.save({ ...enabled, enabled: false });
|
||||
assert.equal(testHarness.read().desiredProfileId, 'profile-2');
|
||||
assert.equal(testHarness.read().profiles[1].desiredServerId, 'server-2');
|
||||
assert.equal(testHarness.service.snapshot().activation, 'passive-loaded');
|
||||
});
|
||||
|
||||
test('significant health states append once without logging every observation', async () => {
|
||||
let clock = 0;
|
||||
const health = { primary: 'unavailable', reserve: 'available' };
|
||||
const events = [];
|
||||
const testHarness = harness(serviceState(), {
|
||||
now: () => new Date(clock),
|
||||
onEvent: (event) => events.push(event),
|
||||
dataplane: {
|
||||
checkConfig: async () => ({}),
|
||||
runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: health[role] }] } }),
|
||||
readFailoverSelector: async () => ({ role: 'primary' }),
|
||||
selectFailoverRole: async (role) => ({ role }),
|
||||
setFailoverActivityEnabled: async () => {},
|
||||
readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date(clock).toISOString() } }),
|
||||
},
|
||||
});
|
||||
await testHarness.service.reconcile();
|
||||
await testHarness.service.runRound();
|
||||
clock = 30_000;
|
||||
await testHarness.service.runRound();
|
||||
health.reserve = 'unavailable';
|
||||
clock = 31_000;
|
||||
await testHarness.service.runRound();
|
||||
clock = 32_000;
|
||||
await testHarness.service.runRound();
|
||||
health.primary = 'available';
|
||||
clock = 33_000;
|
||||
await testHarness.service.runRound();
|
||||
|
||||
assert.deepEqual(events.map(({ type }) => type), [
|
||||
'failover.waiting_for_idle',
|
||||
'failover.both_unhealthy',
|
||||
'failover.recovered',
|
||||
]);
|
||||
});
|
||||
|
||||
test('pending target edits keep selector commits bound to the loaded channel pair', async () => {
|
||||
const state = serviceState();
|
||||
state.failoverPolicy = normalizeFailoverPolicy({
|
||||
...state.failoverPolicy,
|
||||
reserve: { profileId: 'profile-3', serverId: 'server-3' },
|
||||
});
|
||||
state.profiles.push({ id: 'profile-3', servers: [{ id: 'server-3', label: 'Desired later' }] });
|
||||
const testHarness = harness(state, {
|
||||
buildCandidate: () => ({
|
||||
config: {},
|
||||
applied: {
|
||||
...state.appliedFailoverPolicy,
|
||||
reserve: { profileId: 'profile-3', serverId: 'server-3' },
|
||||
reserveConfigFingerprint: 'c'.repeat(64),
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
await testHarness.service.manualSwitch('reserve');
|
||||
assert.equal(testHarness.read().appliedProfileId, 'profile-2');
|
||||
assert.equal(testHarness.read().appliedServerId, 'server-2');
|
||||
assert.equal(testHarness.service.snapshot().activation, 'pending');
|
||||
assert.deepEqual(testHarness.service.snapshot().reserve.target, { profileId: 'profile-2', serverId: 'server-2' });
|
||||
});
|
||||
Reference in New Issue
Block a user