729 lines
28 KiB
TypeScript
729 lines
28 KiB
TypeScript
import crypto from 'node:crypto';
|
|
import {
|
|
createIdleFailoverSnapshot,
|
|
isFailoverConfigured,
|
|
nextFailoverDecision,
|
|
normalizeFailoverPolicy,
|
|
type AppliedFailoverPolicy,
|
|
type FailoverDecisionMemory,
|
|
type FailoverHealth,
|
|
type FailoverPolicy,
|
|
type FailoverRole,
|
|
type FailoverSnapshot,
|
|
} from '../../../shared/failover.js';
|
|
import type { HarborServer, StoredState } from '../../../shared/contracts/state.js';
|
|
import type { ActivityJournalEventInput } from '../../../shared/activityJournal.js';
|
|
import { HarborError } from '../../../shared/errors.js';
|
|
|
|
interface Candidate {
|
|
config: unknown;
|
|
applied: AppliedFailoverPolicy;
|
|
}
|
|
|
|
interface FailoverServiceDependencies {
|
|
state: {
|
|
read(): StoredState;
|
|
update(mutator: (state: StoredState) => Record<string, unknown>): StoredState;
|
|
};
|
|
runtime: { isRunning(): Promise<boolean> };
|
|
dataplane: {
|
|
checkConfig(config: unknown): Promise<unknown>;
|
|
runFailoverProbe(role: FailoverRole, services: unknown, target: string, timeoutMs: number): Promise<unknown>;
|
|
readFailoverSelector(): Promise<unknown>;
|
|
selectFailoverRole(role: FailoverRole): Promise<unknown>;
|
|
setFailoverActivityEnabled(enabled: boolean): Promise<unknown>;
|
|
readFailoverActivity(thresholdBytesPerSecond: number): Promise<unknown>;
|
|
};
|
|
buildCandidate(state: StoredState): Candidate;
|
|
serialize<T>(operation: () => Promise<T>): Promise<T>;
|
|
scheduler?: {
|
|
setTimeout(callback: () => void, intervalMs: number): NodeJS.Timeout;
|
|
clearTimeout(timer: NodeJS.Timeout): void;
|
|
};
|
|
now?: () => Date;
|
|
onWarning?: (error: unknown) => void;
|
|
onSwitch?: (from: FailoverRole, to: FailoverRole, reason: string) => void;
|
|
onEvent?: (event: ActivityJournalEventInput) => void;
|
|
}
|
|
|
|
const record = (value: unknown): Record<string, unknown> => (
|
|
value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}
|
|
);
|
|
|
|
function targetServer(state: StoredState, role: FailoverRole): HarborServer | null {
|
|
const target = state.appliedFailoverPolicy?.[role] || state.failoverPolicy[role];
|
|
return state.profiles.find(({ id }) => id === target.profileId)
|
|
?.servers.find(({ id }) => id === target.serverId) || null;
|
|
}
|
|
|
|
function currentRole(state: StoredState): FailoverRole | null {
|
|
const applied = state.appliedFailoverPolicy;
|
|
if (!applied) return null;
|
|
for (const role of ['primary', 'reserve'] as const) {
|
|
if (
|
|
state.appliedProfileId === applied[role].profileId
|
|
&& state.appliedServerId === applied[role].serverId
|
|
) return role;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function probeHealth(value: unknown): boolean {
|
|
const vpn = record(record(value).vpn);
|
|
const sites = Array.isArray(vpn.sites) ? vpn.sites.map(record) : [];
|
|
return sites.length === 1 && sites[0].status === 'available';
|
|
}
|
|
|
|
function safeErrorCode(error: unknown) {
|
|
const code = error && typeof error === 'object' && 'code' in error ? String(error.code) : '';
|
|
return /^[A-Z0-9_]{1,50}$/.test(code) ? code : 'UNKNOWN';
|
|
}
|
|
|
|
export function createFailoverService(dependencies: FailoverServiceDependencies) {
|
|
const scheduler = dependencies.scheduler || {
|
|
setTimeout: (callback: () => void, intervalMs: number) => setTimeout(callback, intervalMs),
|
|
clearTimeout: (timer: NodeJS.Timeout) => clearTimeout(timer),
|
|
};
|
|
const now = dependencies.now || (() => new Date());
|
|
const epoch = crypto.randomUUID();
|
|
let sequence = 0;
|
|
let generation = 0;
|
|
let timer: NodeJS.Timeout | null = null;
|
|
let collectorEnabled: boolean | null = null;
|
|
let roundPromise: Promise<void> | null = null;
|
|
let roundGeneration: number | null = null;
|
|
let decisionMemory: FailoverDecisionMemory | undefined;
|
|
const healthMemory: Record<FailoverRole, 'healthy' | 'unhealthy' | undefined> = {
|
|
primary: undefined,
|
|
reserve: undefined,
|
|
};
|
|
function clearHealthMemory() {
|
|
healthMemory.primary = undefined;
|
|
healthMemory.reserve = undefined;
|
|
}
|
|
function recordHealthTransition(
|
|
role: FailoverRole,
|
|
health: FailoverHealth,
|
|
capturedGeneration?: number,
|
|
) {
|
|
if (capturedGeneration !== undefined && capturedGeneration !== generation) return;
|
|
if (health !== 'healthy' && health !== 'unhealthy') return;
|
|
const previous = healthMemory[role];
|
|
healthMemory[role] = health;
|
|
if (previous === health) return;
|
|
if (previous === undefined && health === 'healthy') return;
|
|
const type = health === 'unhealthy'
|
|
? role === 'primary' ? 'failover.primary_unavailable' : 'failover.reserve_unavailable'
|
|
: role === 'primary' ? 'failover.primary_recovered' : 'failover.reserve_recovered';
|
|
dependencies.onEvent?.({
|
|
type,
|
|
severity: health === 'unhealthy' ? 'warning' : 'info',
|
|
source: 'failover',
|
|
dedupeKey: null,
|
|
data: {
|
|
role,
|
|
reason: health === 'unhealthy' ? 'probe-failed' : 'probe-recovered',
|
|
},
|
|
});
|
|
}
|
|
|
|
let snapshot = createIdleFailoverSnapshot(dependencies.state.read().failoverPolicy, epoch, sequence);
|
|
|
|
function appliedMatchesDesired(state: StoredState) {
|
|
if (!state.appliedFailoverPolicy) return false;
|
|
try {
|
|
return JSON.stringify(dependencies.buildCandidate(state).applied)
|
|
=== JSON.stringify(state.appliedFailoverPolicy);
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function publish(next: FailoverSnapshot) {
|
|
sequence += 1;
|
|
snapshot = { ...next, observationEpoch: epoch, observationSequence: sequence };
|
|
}
|
|
|
|
function clearTimer() {
|
|
if (timer) scheduler.clearTimeout(timer);
|
|
timer = null;
|
|
}
|
|
|
|
function schedule(delay: number) {
|
|
clearTimer();
|
|
timer = scheduler.setTimeout(() => {
|
|
timer = null;
|
|
void runRound().catch(dependencies.onWarning);
|
|
}, delay);
|
|
timer.unref?.();
|
|
}
|
|
|
|
async function disableCollector() {
|
|
if (collectorEnabled === false) return;
|
|
await dependencies.dataplane.setFailoverActivityEnabled(false);
|
|
collectorEnabled = false;
|
|
}
|
|
|
|
async function deactivate(policy: FailoverPolicy, passiveRole: FailoverRole | null = null) {
|
|
generation += 1;
|
|
clearTimer();
|
|
decisionMemory = undefined;
|
|
clearHealthMemory();
|
|
await disableCollector();
|
|
const idle = createIdleFailoverSnapshot(policy, epoch, sequence);
|
|
if (passiveRole) {
|
|
idle.activation = 'passive-loaded';
|
|
idle.currentRole = passiveRole;
|
|
idle.reason = 'disabled';
|
|
}
|
|
publish(idle);
|
|
}
|
|
|
|
function activeSnapshot(state: StoredState, status: FailoverSnapshot['status'] = 'observing'): FailoverSnapshot {
|
|
const role = state.failoverRuntimeState.reasonCode === 'selector-unknown' ? null : currentRole(state);
|
|
const channel = (target: typeof state.failoverPolicy.primary) => ({
|
|
target,
|
|
health: 'unknown' as FailoverHealth,
|
|
failingServiceIds: [],
|
|
checkedAt: null,
|
|
stateSince: null,
|
|
});
|
|
return {
|
|
observationEpoch: epoch,
|
|
observationSequence: sequence,
|
|
configured: isFailoverConfigured(state.failoverPolicy),
|
|
enabled: state.failoverPolicy.enabled,
|
|
paused: state.failoverPolicy.paused,
|
|
activation: appliedMatchesDesired(state) ? 'active' : 'pending',
|
|
currentRole: role || 'other',
|
|
status,
|
|
primary: channel(state.appliedFailoverPolicy?.primary || state.failoverPolicy.primary),
|
|
reserve: channel(state.appliedFailoverPolicy?.reserve || state.failoverPolicy.reserve),
|
|
nextDecisionAt: null,
|
|
reason: null,
|
|
trafficActivity: null,
|
|
policy: state.failoverPolicy,
|
|
};
|
|
}
|
|
|
|
async function reconcile() {
|
|
let state = dependencies.state.read();
|
|
const policy = state.failoverPolicy;
|
|
if (!policy.enabled) {
|
|
const role = currentRole(state);
|
|
const passiveRole = role && await dependencies.runtime.isRunning() ? role : null;
|
|
return deactivate(policy, passiveRole);
|
|
}
|
|
const running = await dependencies.runtime.isRunning();
|
|
if (!running || !state.appliedFailoverPolicy || !currentRole(state)) {
|
|
generation += 1;
|
|
clearHealthMemory();
|
|
clearTimer();
|
|
await disableCollector();
|
|
const pending = activeSnapshot(state, 'idle');
|
|
pending.activation = running ? 'pending' : 'inactive';
|
|
pending.reason = running ? 'pending-activation' : 'vpn-stopped';
|
|
publish(pending);
|
|
return;
|
|
}
|
|
const role = currentRole(state)!;
|
|
const selected = record(await dependencies.dataplane.readFailoverSelector());
|
|
if (selected.role !== role) await dependencies.dataplane.selectFailoverRole(role);
|
|
if (state.failoverRuntimeState.reasonCode === 'selector-unknown') {
|
|
state = dependencies.state.update((current) => ({
|
|
...current,
|
|
failoverRuntimeState: { ...current.failoverRuntimeState, reasonCode: null },
|
|
}));
|
|
}
|
|
if (collectorEnabled !== true) {
|
|
await dependencies.dataplane.setFailoverActivityEnabled(true);
|
|
collectorEnabled = true;
|
|
}
|
|
generation += 1;
|
|
const active = activeSnapshot(state);
|
|
if (active.activation === 'pending') active.reason = 'pending-activation';
|
|
publish(active);
|
|
schedule(0);
|
|
}
|
|
|
|
async function reconcileAfterCommit() {
|
|
try {
|
|
await reconcile();
|
|
} catch (error) {
|
|
dependencies.onWarning?.(error);
|
|
const failed = activeSnapshot(dependencies.state.read(), 'error');
|
|
failed.reason = 'reconcile-failed';
|
|
publish(failed);
|
|
}
|
|
}
|
|
|
|
async function assessRole(role: FailoverRole, policy: FailoverPolicy) {
|
|
const custom = dependencies.state.read().diagnostics.customServices;
|
|
const results = await Promise.all(policy.checks.map(async (check) => {
|
|
try {
|
|
return {
|
|
id: check.serviceId,
|
|
ok: probeHealth(await dependencies.dataplane.runFailoverProbe(
|
|
role,
|
|
custom,
|
|
`site:${check.serviceId}`,
|
|
check.timeoutMs,
|
|
)),
|
|
};
|
|
} catch {
|
|
return { id: check.serviceId, ok: null };
|
|
}
|
|
}));
|
|
return {
|
|
health: results.some(({ ok }) => ok === null)
|
|
? 'unknown' as const
|
|
: results.every(({ ok }) => ok) ? 'healthy' as const : 'unhealthy' as const,
|
|
failingServiceIds: results.filter(({ ok }) => ok === false).map(({ id }) => id),
|
|
};
|
|
}
|
|
|
|
async function switchWithinQueue(role: FailoverRole, reason: string) {
|
|
const before = dependencies.state.read();
|
|
const from = currentRole(before);
|
|
if (!from || from === role) return;
|
|
try {
|
|
await dependencies.dataplane.selectFailoverRole(role);
|
|
const server = targetServer(before, role);
|
|
if (!server) throw new HarborError('SERVER_NOT_FOUND');
|
|
const target = before.appliedFailoverPolicy![role];
|
|
const switchedAt = now().toISOString();
|
|
const cutoff = now().getTime() - before.failoverPolicy.flapProtection.windowMs;
|
|
const history = role === 'reserve'
|
|
? [...before.failoverRuntimeState.failoverHistory.filter((value) => Date.parse(value) >= cutoff), switchedAt]
|
|
: before.failoverRuntimeState.failoverHistory.filter((value) => Date.parse(value) >= cutoff);
|
|
const quarantine = history.length >= before.failoverPolicy.flapProtection.count
|
|
? new Date(now().getTime() + before.failoverPolicy.flapProtection.quarantineMs).toISOString()
|
|
: before.failoverRuntimeState.primaryQuarantineUntil;
|
|
dependencies.state.update((state) => ({
|
|
...state,
|
|
appliedProfileId: target.profileId,
|
|
appliedServerId: target.serverId,
|
|
appliedServerSnapshot: server,
|
|
failoverPolicy: reason === 'manual'
|
|
? state.failoverPolicy
|
|
: role === 'primary' ? { ...state.failoverPolicy, paused: false } : state.failoverPolicy,
|
|
failoverRuntimeState: {
|
|
...state.failoverRuntimeState,
|
|
lastSwitchAt: switchedAt,
|
|
holdUntil: role === 'reserve'
|
|
? new Date(now().getTime() + state.failoverPolicy.minimumReserveMs).toISOString()
|
|
: null,
|
|
primaryQuarantineUntil: quarantine,
|
|
failoverHistory: history,
|
|
reasonCode: reason,
|
|
},
|
|
}));
|
|
} catch (error) {
|
|
try {
|
|
await dependencies.dataplane.selectFailoverRole(from);
|
|
} catch (rollback) {
|
|
generation += 1;
|
|
clearTimer();
|
|
decisionMemory = undefined;
|
|
dependencies.state.update((state) => ({
|
|
...state,
|
|
failoverPolicy: { ...state.failoverPolicy, paused: true },
|
|
failoverRuntimeState: { ...state.failoverRuntimeState, reasonCode: 'selector-unknown' },
|
|
}));
|
|
const failed = activeSnapshot(dependencies.state.read(), 'error');
|
|
failed.reason = 'selector-unknown';
|
|
publish(failed);
|
|
throw new AggregateError([error, rollback], 'Failover selector rollback failed');
|
|
}
|
|
throw error;
|
|
}
|
|
dependencies.onSwitch?.(from, role, reason);
|
|
dependencies.onEvent?.({
|
|
type: 'failover.switched',
|
|
severity: 'info',
|
|
source: 'failover',
|
|
dedupeKey: `failover.switched:${dependencies.state.read().revision}`,
|
|
data: {
|
|
fromRole: from,
|
|
toRole: role,
|
|
primaryLabel: targetServer(dependencies.state.read(), 'primary')?.label || 'Primary',
|
|
reserveLabel: targetServer(dependencies.state.read(), 'reserve')?.label || 'Reserve',
|
|
reason,
|
|
manual: reason === 'manual',
|
|
},
|
|
});
|
|
}
|
|
|
|
async function performRound(capturedGeneration: number) {
|
|
const prepared = await dependencies.serialize(async () => {
|
|
const state = dependencies.state.read();
|
|
const role = currentRole(state);
|
|
if (
|
|
capturedGeneration !== generation
|
|
|| !state.failoverPolicy.enabled
|
|
|| !state.appliedFailoverPolicy
|
|
|| !role
|
|
) return null;
|
|
const selected = record(await dependencies.dataplane.readFailoverSelector());
|
|
if (selected.role !== role) await dependencies.dataplane.selectFailoverRole(role);
|
|
await dependencies.dataplane.setFailoverActivityEnabled(true);
|
|
collectorEnabled = true;
|
|
const latest = dependencies.state.read();
|
|
return capturedGeneration === generation
|
|
&& latest.failoverPolicy.enabled
|
|
&& currentRole(latest) === role
|
|
? { state: latest, policy: latest.failoverPolicy, role }
|
|
: null;
|
|
});
|
|
if (!prepared) return;
|
|
const { state, policy, role } = prepared;
|
|
const checkedAt = now().toISOString();
|
|
const previousSnapshot = snapshot;
|
|
publish({ ...snapshot, reason: 'checking-channels' });
|
|
let primary;
|
|
let reserve;
|
|
try {
|
|
[primary, reserve] = await Promise.all([
|
|
assessRole('primary', policy),
|
|
assessRole('reserve', policy),
|
|
]);
|
|
} catch {
|
|
primary = { health: 'unknown' as const, failingServiceIds: [] };
|
|
reserve = { health: 'unknown' as const, failingServiceIds: [] };
|
|
}
|
|
if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return;
|
|
recordHealthTransition('primary', primary.health, capturedGeneration);
|
|
recordHealthTransition('reserve', reserve.health, capturedGeneration);
|
|
const activityResponse = record(await dependencies.dataplane.readFailoverActivity(
|
|
policy.trafficGuard.thresholdBytesPerSecond,
|
|
));
|
|
if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return;
|
|
const activity = record(activityResponse.activity);
|
|
const observedAt = typeof activity.observedAt === 'string' ? Date.parse(activity.observedAt) : NaN;
|
|
const activityState = Number.isFinite(observedAt) && now().getTime() - observedAt <= 4_000
|
|
&& (activity.state === 'active' || activity.state === 'quiet')
|
|
? activity.state
|
|
: 'unknown';
|
|
const decision = nextFailoverDecision({
|
|
now: now().getTime(),
|
|
policy,
|
|
currentRole: role,
|
|
primaryHealth: primary.health,
|
|
reserveHealth: reserve.health,
|
|
activity: activityState,
|
|
holdUntil: Date.parse(state.failoverRuntimeState.holdUntil || '') || null,
|
|
primaryQuarantineUntil: Date.parse(state.failoverRuntimeState.primaryQuarantineUntil || '') || null,
|
|
memory: decisionMemory,
|
|
});
|
|
decisionMemory = decision.memory;
|
|
const next = activeSnapshot(state, decision.status);
|
|
next.currentRole = role;
|
|
next.primary = {
|
|
...next.primary,
|
|
...primary,
|
|
checkedAt,
|
|
stateSince: previousSnapshot.primary.health === primary.health
|
|
? previousSnapshot.primary.stateSince || checkedAt
|
|
: checkedAt,
|
|
};
|
|
next.reserve = {
|
|
...next.reserve,
|
|
...reserve,
|
|
checkedAt,
|
|
stateSince: previousSnapshot.reserve.health === reserve.health
|
|
? previousSnapshot.reserve.stateSince || checkedAt
|
|
: checkedAt,
|
|
};
|
|
next.reason = decision.reason;
|
|
next.nextDecisionAt = decision.nextDecisionAt ? new Date(decision.nextDecisionAt).toISOString() : null;
|
|
next.trafficActivity = activityState === 'unknown' ? {
|
|
state: 'unknown',
|
|
observedAt: Number.isFinite(observedAt) ? new Date(observedAt).toISOString() : checkedAt,
|
|
windowMs: 10_000,
|
|
thresholdBytesPerSecond: policy.trafficGuard.thresholdBytesPerSecond,
|
|
totalBytesPerSecond: 0,
|
|
transmittingConnections: 0,
|
|
quietSince: null,
|
|
switchTarget: decision.switchTo,
|
|
blockers: [],
|
|
} : {
|
|
state: activityState,
|
|
observedAt: String(activity.observedAt),
|
|
windowMs: Number(activity.windowMs) || 10_000,
|
|
thresholdBytesPerSecond: policy.trafficGuard.thresholdBytesPerSecond,
|
|
totalBytesPerSecond: Number(activity.totalBytesPerSecond) || 0,
|
|
transmittingConnections: Number(activity.transmittingConnections) || 0,
|
|
quietSince: typeof activity.quietSince === 'string' ? activity.quietSince : null,
|
|
switchTarget: decision.switchTo,
|
|
blockers: (Array.isArray(activity.blockers) ? activity.blockers : []).slice(0, 3) as FailoverSnapshot['trafficActivity'] extends infer T ? T extends { blockers: infer B } ? B : never : never,
|
|
};
|
|
publish(next);
|
|
if (decision.status === 'waiting-for-idle' && previousSnapshot.status !== 'waiting-for-idle') {
|
|
dependencies.onEvent?.({
|
|
type: 'failover.waiting_for_idle',
|
|
severity: 'info',
|
|
source: 'failover',
|
|
dedupeKey: `failover.waiting_for_idle:${state.revision}:${role}:${decision.reason}`,
|
|
data: { fromRole: role, toRole: decision.switchTo || (role === 'primary' ? 'reserve' : 'primary'), reason: decision.reason },
|
|
});
|
|
}
|
|
if (decision.reason === 'both-unhealthy' && previousSnapshot.reason !== 'both-unhealthy') {
|
|
dependencies.onEvent?.({
|
|
type: 'failover.both_unhealthy',
|
|
severity: 'warning',
|
|
source: 'failover',
|
|
dedupeKey: `failover.both_unhealthy:${state.revision}`,
|
|
data: { reason: decision.reason },
|
|
});
|
|
}
|
|
if (decision.switchTo) {
|
|
try {
|
|
const switched = await dependencies.serialize(async () => {
|
|
const current = dependencies.state.read();
|
|
if (
|
|
capturedGeneration !== generation
|
|
|| !current.failoverPolicy.enabled
|
|
|| current.failoverPolicy.paused
|
|
|| currentRole(current) !== role
|
|
|| !current.appliedFailoverPolicy
|
|
) return false;
|
|
let freshPrimary;
|
|
let freshReserve;
|
|
try {
|
|
[freshPrimary, freshReserve] = await Promise.all([
|
|
assessRole('primary', current.failoverPolicy),
|
|
assessRole('reserve', current.failoverPolicy),
|
|
]);
|
|
} catch {
|
|
return false;
|
|
}
|
|
const healthStillAllowsSwitch = decision.switchTo === 'reserve'
|
|
? freshPrimary.health === 'unhealthy' && freshReserve.health === 'healthy'
|
|
: freshPrimary.health === 'healthy';
|
|
if (!healthStillAllowsSwitch || capturedGeneration !== generation) return false;
|
|
if (current.failoverPolicy.trafficGuard.enabled) {
|
|
const freshResponse = record(await dependencies.dataplane.readFailoverActivity(
|
|
current.failoverPolicy.trafficGuard.thresholdBytesPerSecond,
|
|
));
|
|
const freshActivity = record(freshResponse.activity);
|
|
const freshObservedAt = typeof freshActivity.observedAt === 'string'
|
|
? Date.parse(freshActivity.observedAt)
|
|
: NaN;
|
|
const freshQuietSince = typeof freshActivity.quietSince === 'string'
|
|
? Date.parse(freshActivity.quietSince)
|
|
: NaN;
|
|
if (
|
|
capturedGeneration !== generation
|
|
|| freshActivity.state !== 'quiet'
|
|
|| !Number.isFinite(freshObservedAt)
|
|
|| now().getTime() - freshObservedAt > 4_000
|
|
|| !Number.isFinite(freshQuietSince)
|
|
|| now().getTime() - freshQuietSince < current.failoverPolicy.trafficGuard.quietWindowMs
|
|
) return false;
|
|
}
|
|
await switchWithinQueue(decision.switchTo!, decision.reason);
|
|
return true;
|
|
});
|
|
if (!switched) {
|
|
const cancelled = activeSnapshot(dependencies.state.read(), 'observing');
|
|
cancelled.reason = 'revalidation-required';
|
|
publish(cancelled);
|
|
decisionMemory = undefined;
|
|
return;
|
|
}
|
|
} catch (error) {
|
|
const failed = activeSnapshot(dependencies.state.read(), 'error');
|
|
failed.reason = dependencies.state.read().failoverRuntimeState.reasonCode === 'selector-unknown'
|
|
? 'selector-unknown'
|
|
: 'switch-failed';
|
|
publish(failed);
|
|
dependencies.onEvent?.({
|
|
type: 'failover.switch_failed',
|
|
severity: 'error',
|
|
source: 'failover',
|
|
dedupeKey: `failover.switch_failed:${state.revision}:${role}:${decision.switchTo}`,
|
|
data: { fromRole: role, toRole: decision.switchTo, reason: decision.reason, errorCode: safeErrorCode(error) },
|
|
});
|
|
throw error;
|
|
}
|
|
if (capturedGeneration !== generation) return;
|
|
publish(activeSnapshot(dependencies.state.read(), decision.switchTo === 'reserve' ? 'reserve' : 'primary'));
|
|
decisionMemory = undefined;
|
|
}
|
|
}
|
|
|
|
async function runRound(): Promise<void> {
|
|
if (roundPromise) {
|
|
const pending = roundPromise;
|
|
if (roundGeneration === generation) return pending;
|
|
try {
|
|
await pending;
|
|
} catch {
|
|
// The original caller owns the stale round error; continue with current-generation work.
|
|
}
|
|
if (roundPromise && roundPromise !== pending) return roundPromise;
|
|
return runRound();
|
|
}
|
|
const capturedGeneration = generation;
|
|
let trackedPromise: Promise<void>;
|
|
trackedPromise = performRound(capturedGeneration).finally(() => {
|
|
if (roundPromise === trackedPromise) {
|
|
roundPromise = null;
|
|
roundGeneration = null;
|
|
}
|
|
if (capturedGeneration === generation && snapshot.reason === 'checking-channels') {
|
|
const failed = activeSnapshot(dependencies.state.read(), 'error');
|
|
failed.reason = 'health-unknown';
|
|
publish(failed);
|
|
}
|
|
const policy = dependencies.state.read().failoverPolicy;
|
|
if (capturedGeneration === generation && policy.enabled && dependencies.state.read().appliedFailoverPolicy) {
|
|
const decisionAt = snapshot.nextDecisionAt ? Date.parse(snapshot.nextDecisionAt) : NaN;
|
|
const decisionDelay = Number.isFinite(decisionAt)
|
|
? Math.max(250, decisionAt - now().getTime())
|
|
: policy.intervalMs;
|
|
schedule(Math.min(policy.intervalMs, decisionDelay));
|
|
}
|
|
});
|
|
roundPromise = trackedPromise;
|
|
roundGeneration = capturedGeneration;
|
|
return trackedPromise;
|
|
}
|
|
|
|
function save(value: unknown) {
|
|
return dependencies.serialize(async () => {
|
|
let policy: FailoverPolicy;
|
|
try {
|
|
policy = normalizeFailoverPolicy(value, { strict: true });
|
|
} catch (cause) {
|
|
throw new HarborError('REQUEST_INVALID', { cause });
|
|
}
|
|
if (policy.enabled && !isFailoverConfigured(policy)) throw new HarborError('REQUEST_INVALID');
|
|
const before = dependencies.state.read();
|
|
const candidateState = { ...before, failoverPolicy: policy };
|
|
if (policy.enabled) {
|
|
const candidate = dependencies.buildCandidate(candidateState);
|
|
await dependencies.dataplane.checkConfig(candidate.config);
|
|
}
|
|
dependencies.state.update((state) => {
|
|
const role = before.failoverPolicy.enabled && !policy.enabled ? currentRole(state) : null;
|
|
const target = role ? state.appliedFailoverPolicy?.[role] : null;
|
|
return {
|
|
...state,
|
|
failoverPolicy: policy,
|
|
...(target ? {
|
|
desiredProfileId: target.profileId,
|
|
profiles: state.profiles.map((profile) => profile.id === target.profileId
|
|
? { ...profile, desiredServerId: target.serverId }
|
|
: profile),
|
|
} : {}),
|
|
};
|
|
});
|
|
decisionMemory = undefined;
|
|
if (before.failoverPolicy.enabled !== policy.enabled) clearHealthMemory();
|
|
if (before.failoverPolicy.enabled !== policy.enabled) {
|
|
dependencies.onEvent?.({
|
|
type: policy.enabled ? 'failover.enabled' : 'failover.disabled',
|
|
severity: 'info',
|
|
source: 'failover',
|
|
dedupeKey: `failover.${policy.enabled ? 'enabled' : 'disabled'}:${dependencies.state.read().revision}`,
|
|
data: {},
|
|
});
|
|
}
|
|
await reconcileAfterCommit();
|
|
});
|
|
}
|
|
|
|
function pause(paused: boolean) {
|
|
return dependencies.serialize(async () => {
|
|
const before = dependencies.state.read();
|
|
if (!paused && (
|
|
!before.appliedFailoverPolicy
|
|
|| !targetServer(before, 'primary')
|
|
|| !targetServer(before, 'reserve')
|
|
)) throw new HarborError('REQUEST_INVALID');
|
|
dependencies.state.update((state) => ({
|
|
...state,
|
|
failoverPolicy: { ...state.failoverPolicy, paused },
|
|
}));
|
|
decisionMemory = undefined;
|
|
dependencies.onEvent?.({
|
|
type: paused ? 'failover.paused' : 'failover.resumed',
|
|
severity: 'info',
|
|
source: 'failover',
|
|
dedupeKey: `failover.${paused ? 'paused' : 'resumed'}:${dependencies.state.read().revision}`,
|
|
data: {},
|
|
});
|
|
await reconcileAfterCommit();
|
|
});
|
|
}
|
|
|
|
async function manualSwitch(role: FailoverRole) {
|
|
await dependencies.serialize(async () => {
|
|
const state = dependencies.state.read();
|
|
if (!state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !currentRole(state)) {
|
|
throw new HarborError('REQUEST_INVALID');
|
|
}
|
|
await switchWithinQueue(role, 'manual');
|
|
});
|
|
await reconcileAfterCommit();
|
|
}
|
|
|
|
const prepareActivation = (role: FailoverRole) => dependencies.dataplane.selectFailoverRole(role);
|
|
|
|
async function restoreAppliedActivation(state: StoredState) {
|
|
if (!state.appliedFailoverPolicy) return;
|
|
const role = currentRole(state);
|
|
if (!role) throw new HarborError('CONFIG_INVALID');
|
|
await prepareActivation(role);
|
|
}
|
|
|
|
function checkNow() {
|
|
return dependencies.serialize(async () => {
|
|
const state = dependencies.state.read();
|
|
if (!state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !currentRole(state)) {
|
|
throw new HarborError('REQUEST_INVALID');
|
|
}
|
|
const capturedGeneration = ++generation;
|
|
clearTimer();
|
|
const checkedAt = now().toISOString();
|
|
publish({ ...snapshot, reason: 'checking-channels' });
|
|
let primary;
|
|
let reserve;
|
|
try {
|
|
[primary, reserve] = await Promise.all([
|
|
assessRole('primary', state.failoverPolicy),
|
|
assessRole('reserve', state.failoverPolicy),
|
|
]);
|
|
} catch {
|
|
primary = { health: 'unknown' as const, failingServiceIds: [] };
|
|
reserve = { health: 'unknown' as const, failingServiceIds: [] };
|
|
}
|
|
if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return;
|
|
recordHealthTransition('primary', primary.health, capturedGeneration);
|
|
recordHealthTransition('reserve', reserve.health, capturedGeneration);
|
|
const next = activeSnapshot(dependencies.state.read(), 'observing');
|
|
next.primary = { ...next.primary, ...primary, checkedAt, stateSince: checkedAt };
|
|
next.reserve = { ...next.reserve, ...reserve, checkedAt, stateSince: checkedAt };
|
|
next.reason = 'manual-check';
|
|
publish(next);
|
|
schedule(state.failoverPolicy.intervalMs);
|
|
});
|
|
}
|
|
|
|
return {
|
|
snapshot: () => snapshot,
|
|
save,
|
|
pause,
|
|
manualSwitch,
|
|
checkNow,
|
|
prepareActivation,
|
|
restoreAppliedActivation,
|
|
reconcile,
|
|
runRound,
|
|
shutdown: () => deactivate(dependencies.state.read().failoverPolicy),
|
|
};
|
|
}
|
|
|
|
export type FailoverService = ReturnType<typeof createFailoverService>;
|