356 lines
13 KiB
TypeScript
356 lines
13 KiB
TypeScript
export type FailoverRole = 'primary' | 'reserve';
|
|
export type FailoverHealth = 'healthy' | 'unhealthy' | 'unknown' | 'not-monitoring';
|
|
|
|
export interface FailoverTarget {
|
|
profileId: string;
|
|
serverId: string;
|
|
}
|
|
|
|
export interface FailoverCheck {
|
|
serviceId: string;
|
|
timeoutMs: number;
|
|
}
|
|
|
|
export interface FailoverPolicy {
|
|
version: 1;
|
|
enabled: boolean;
|
|
paused: boolean;
|
|
primary: FailoverTarget;
|
|
reserve: FailoverTarget;
|
|
checks: FailoverCheck[];
|
|
intervalMs: number;
|
|
failureWindowMs: number;
|
|
recoveryWindowMs: number;
|
|
trafficGuard: {
|
|
enabled: boolean;
|
|
thresholdBytesPerSecond: number;
|
|
quietWindowMs: number;
|
|
};
|
|
minimumReserveMs: number;
|
|
flapProtection: {
|
|
count: number;
|
|
windowMs: number;
|
|
quarantineMs: number;
|
|
};
|
|
}
|
|
|
|
export interface FailoverRuntimeState {
|
|
lastSwitchAt: string | null;
|
|
holdUntil: string | null;
|
|
primaryQuarantineUntil: string | null;
|
|
failoverHistory: string[];
|
|
reasonCode: string | null;
|
|
}
|
|
|
|
export interface AppliedFailoverPolicy {
|
|
primary: FailoverTarget;
|
|
reserve: FailoverTarget;
|
|
primaryConfigFingerprint: string;
|
|
reserveConfigFingerprint: string;
|
|
}
|
|
|
|
export interface FailoverActivity {
|
|
state: 'active' | 'quiet' | 'unknown';
|
|
observedAt: string;
|
|
windowMs: number;
|
|
thresholdBytesPerSecond: number;
|
|
totalBytesPerSecond: number;
|
|
transmittingConnections: number;
|
|
quietSince: string | null;
|
|
switchTarget: FailoverRole | null;
|
|
blockers: Array<{
|
|
device: string;
|
|
service: string;
|
|
uploadBytesPerSecond: number;
|
|
downloadBytesPerSecond: number;
|
|
}>;
|
|
}
|
|
|
|
export interface FailoverSnapshot {
|
|
observationEpoch: string;
|
|
observationSequence: number;
|
|
configured: boolean;
|
|
enabled: boolean;
|
|
paused: boolean;
|
|
activation: 'inactive' | 'active' | 'pending' | 'passive-loaded';
|
|
currentRole: FailoverRole | 'other' | 'none';
|
|
status: 'idle' | 'observing' | 'primary' | 'reserve' | 'waiting-for-idle' | 'blocked' | 'switching' | 'error';
|
|
primary: { target: FailoverTarget; health: FailoverHealth; failingServiceIds: string[]; checkedAt: string | null; stateSince: string | null };
|
|
reserve: { target: FailoverTarget; health: FailoverHealth; failingServiceIds: string[]; checkedAt: string | null; stateSince: string | null };
|
|
nextDecisionAt: string | null;
|
|
reason: string | null;
|
|
trafficActivity: FailoverActivity | null;
|
|
policy: FailoverPolicy;
|
|
}
|
|
|
|
export interface FailoverDecisionMemory {
|
|
primaryFailedSince: number | null;
|
|
primaryRecoveredSince: number | null;
|
|
quietSince: number | null;
|
|
}
|
|
|
|
export interface FailoverDecisionInput {
|
|
now: number;
|
|
policy: FailoverPolicy;
|
|
currentRole: FailoverRole;
|
|
primaryHealth: Exclude<FailoverHealth, 'not-monitoring'>;
|
|
reserveHealth: Exclude<FailoverHealth, 'not-monitoring'>;
|
|
activity: 'active' | 'quiet' | 'unknown';
|
|
holdUntil?: number | null;
|
|
primaryQuarantineUntil?: number | null;
|
|
memory?: FailoverDecisionMemory;
|
|
}
|
|
|
|
export interface FailoverDecision {
|
|
status: FailoverSnapshot['status'];
|
|
reason: string;
|
|
switchTo: FailoverRole | null;
|
|
memory: FailoverDecisionMemory;
|
|
nextDecisionAt: number | null;
|
|
}
|
|
|
|
const text = (value: unknown) => typeof value === 'string' ? value.trim() : '';
|
|
const record = (value: unknown): Record<string, unknown> => (
|
|
value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}
|
|
);
|
|
const bounded = (value: unknown, fallback: number, minimum: number, maximum: number) => {
|
|
const candidate = Number(value);
|
|
return Number.isFinite(candidate) ? Math.min(maximum, Math.max(minimum, Math.round(candidate))) : fallback;
|
|
};
|
|
const equivalent = (left: unknown, right: unknown): boolean => {
|
|
if (Array.isArray(left) || Array.isArray(right)) {
|
|
return Array.isArray(left) && Array.isArray(right)
|
|
&& left.length === right.length
|
|
&& left.every((value, index) => equivalent(value, right[index]));
|
|
}
|
|
if (left && right && typeof left === 'object' && typeof right === 'object') {
|
|
const leftRecord = left as Record<string, unknown>;
|
|
const rightRecord = right as Record<string, unknown>;
|
|
const leftKeys = Object.keys(leftRecord).sort();
|
|
const rightKeys = Object.keys(rightRecord).sort();
|
|
return leftKeys.length === rightKeys.length
|
|
&& leftKeys.every((key, index) => key === rightKeys[index] && equivalent(leftRecord[key], rightRecord[key]));
|
|
}
|
|
return Object.is(left, right);
|
|
};
|
|
|
|
export const DEFAULT_FAILOVER_POLICY: FailoverPolicy = Object.freeze({
|
|
version: 1,
|
|
enabled: false,
|
|
paused: false,
|
|
primary: { profileId: '', serverId: '' },
|
|
reserve: { profileId: '', serverId: '' },
|
|
checks: [{ serviceId: 'youtube', timeoutMs: 6_000 }, { serviceId: 'google', timeoutMs: 6_000 }],
|
|
intervalMs: 60_000,
|
|
failureWindowMs: 120_000,
|
|
recoveryWindowMs: 900_000,
|
|
trafficGuard: { enabled: true, thresholdBytesPerSecond: 32 * 1024, quietWindowMs: 30_000 },
|
|
minimumReserveMs: 600_000,
|
|
flapProtection: { count: 3, windowMs: 86_400_000, quarantineMs: 21_600_000 },
|
|
});
|
|
|
|
export const DEFAULT_FAILOVER_RUNTIME_STATE: FailoverRuntimeState = Object.freeze({
|
|
lastSwitchAt: null,
|
|
holdUntil: null,
|
|
primaryQuarantineUntil: null,
|
|
failoverHistory: [],
|
|
reasonCode: null,
|
|
});
|
|
|
|
function target(value: unknown): FailoverTarget {
|
|
const candidate = record(value);
|
|
return { profileId: text(candidate.profileId), serverId: text(candidate.serverId) };
|
|
}
|
|
|
|
export function normalizeFailoverPolicy(value: unknown, { strict = false } = {}): FailoverPolicy {
|
|
const candidate = record(value);
|
|
const requestedChecks = Array.isArray(candidate.checks) ? candidate.checks : DEFAULT_FAILOVER_POLICY.checks;
|
|
const checks = requestedChecks.map((value) => {
|
|
const check = record(value);
|
|
return {
|
|
serviceId: text(check.serviceId).slice(0, 100),
|
|
timeoutMs: bounded(check.timeoutMs, 6_000, 2_000, 30_000),
|
|
};
|
|
}).filter(({ serviceId }, index, all) => serviceId && all.findIndex((item) => item.serviceId === serviceId) === index).slice(0, 10);
|
|
const trafficGuard = record(candidate.trafficGuard);
|
|
const flapProtection = record(candidate.flapProtection);
|
|
const policy: FailoverPolicy = {
|
|
version: 1,
|
|
enabled: candidate.enabled === true,
|
|
paused: candidate.paused === true,
|
|
primary: target(candidate.primary),
|
|
reserve: target(candidate.reserve),
|
|
checks,
|
|
intervalMs: bounded(candidate.intervalMs, 60_000, 15_000, 900_000),
|
|
failureWindowMs: 0,
|
|
recoveryWindowMs: bounded(candidate.recoveryWindowMs, 900_000, 60_000, 86_400_000),
|
|
trafficGuard: {
|
|
enabled: trafficGuard.enabled !== false,
|
|
thresholdBytesPerSecond: bounded(trafficGuard.thresholdBytesPerSecond, 32 * 1024, 1024, 100 * 1024 * 1024),
|
|
quietWindowMs: bounded(trafficGuard.quietWindowMs, 30_000, 5_000, 600_000),
|
|
},
|
|
minimumReserveMs: bounded(candidate.minimumReserveMs, 600_000, 60_000, 86_400_000),
|
|
flapProtection: {
|
|
count: bounded(flapProtection.count, 3, 2, 10),
|
|
windowMs: bounded(flapProtection.windowMs, 86_400_000, 3_600_000, 72 * 3_600_000),
|
|
quarantineMs: bounded(flapProtection.quarantineMs, 21_600_000, 600_000, 7 * 86_400_000),
|
|
},
|
|
};
|
|
policy.failureWindowMs = bounded(
|
|
candidate.failureWindowMs,
|
|
120_000,
|
|
policy.intervalMs * 2,
|
|
1_800_000,
|
|
);
|
|
if (strict && !equivalent(policy, value)) {
|
|
throw new TypeError('Invalid failover policy');
|
|
}
|
|
return policy;
|
|
}
|
|
|
|
export function normalizeFailoverRuntimeState(value: unknown): FailoverRuntimeState {
|
|
const candidate = record(value);
|
|
const date = (value: unknown) => typeof value === 'string' && Number.isFinite(Date.parse(value)) ? value : null;
|
|
return {
|
|
lastSwitchAt: date(candidate.lastSwitchAt),
|
|
holdUntil: date(candidate.holdUntil),
|
|
primaryQuarantineUntil: date(candidate.primaryQuarantineUntil),
|
|
failoverHistory: (Array.isArray(candidate.failoverHistory) ? candidate.failoverHistory : [])
|
|
.map(date).filter((item): item is string => Boolean(item)).slice(-20),
|
|
reasonCode: text(candidate.reasonCode) || null,
|
|
};
|
|
}
|
|
|
|
export function normalizeAppliedFailoverPolicy(value: unknown): AppliedFailoverPolicy | null {
|
|
if (!value) return null;
|
|
const candidate = record(value);
|
|
const primary = target(candidate.primary);
|
|
const reserve = target(candidate.reserve);
|
|
const primaryConfigFingerprint = text(candidate.primaryConfigFingerprint);
|
|
const reserveConfigFingerprint = text(candidate.reserveConfigFingerprint);
|
|
return primary.profileId && primary.serverId && reserve.profileId && reserve.serverId
|
|
&& /^[a-f0-9]{64}$/.test(primaryConfigFingerprint)
|
|
&& /^[a-f0-9]{64}$/.test(reserveConfigFingerprint)
|
|
? { primary, reserve, primaryConfigFingerprint, reserveConfigFingerprint }
|
|
: null;
|
|
}
|
|
|
|
export function isFailoverConfigured(policy: FailoverPolicy) {
|
|
return Boolean(
|
|
policy.primary.profileId && policy.primary.serverId
|
|
&& policy.reserve.profileId && policy.reserve.serverId
|
|
&& policy.checks.length
|
|
&& (policy.primary.profileId !== policy.reserve.profileId
|
|
|| policy.primary.serverId !== policy.reserve.serverId)
|
|
);
|
|
}
|
|
|
|
export function createIdleFailoverSnapshot(
|
|
policy: FailoverPolicy,
|
|
observationEpoch = '',
|
|
observationSequence = 0,
|
|
): FailoverSnapshot {
|
|
const channel = (target: FailoverTarget) => ({
|
|
target,
|
|
health: 'not-monitoring' as const,
|
|
failingServiceIds: [],
|
|
checkedAt: null,
|
|
stateSince: null,
|
|
});
|
|
return {
|
|
observationEpoch,
|
|
observationSequence,
|
|
configured: isFailoverConfigured(policy),
|
|
enabled: policy.enabled,
|
|
paused: policy.paused,
|
|
activation: 'inactive',
|
|
currentRole: 'none',
|
|
status: 'idle',
|
|
primary: channel(policy.primary),
|
|
reserve: channel(policy.reserve),
|
|
nextDecisionAt: null,
|
|
reason: null,
|
|
trafficActivity: null,
|
|
policy,
|
|
};
|
|
}
|
|
|
|
export function nextFailoverDecision(input: FailoverDecisionInput): FailoverDecision {
|
|
const memory = input.memory || {
|
|
primaryFailedSince: null,
|
|
primaryRecoveredSince: null,
|
|
quietSince: null,
|
|
};
|
|
const next = { ...memory };
|
|
if (!input.policy.enabled || input.policy.paused) {
|
|
return { status: 'idle', reason: input.policy.paused ? 'paused' : 'disabled', switchTo: null, memory: next, nextDecisionAt: null };
|
|
}
|
|
if (input.primaryHealth === 'unhealthy' && input.reserveHealth === 'unhealthy') {
|
|
next.quietSince = null;
|
|
return { status: 'blocked', reason: 'both-unhealthy', switchTo: null, memory: next, nextDecisionAt: null };
|
|
}
|
|
|
|
let target: FailoverRole | null = null;
|
|
let readyAt: number | null = null;
|
|
if (input.currentRole === 'primary') {
|
|
next.primaryRecoveredSince = null;
|
|
if (input.primaryHealth !== 'unhealthy') {
|
|
next.primaryFailedSince = null;
|
|
next.quietSince = null;
|
|
return { status: 'primary', reason: input.primaryHealth === 'healthy' ? 'primary-healthy' : 'health-unknown', switchTo: null, memory: next, nextDecisionAt: null };
|
|
}
|
|
next.primaryFailedSince ??= input.now;
|
|
readyAt = next.primaryFailedSince + input.policy.failureWindowMs;
|
|
if (input.now < readyAt || input.reserveHealth !== 'healthy') {
|
|
next.quietSince = null;
|
|
return { status: 'observing', reason: input.reserveHealth === 'healthy' ? 'failure-window' : 'reserve-not-healthy', switchTo: null, memory: next, nextDecisionAt: readyAt };
|
|
}
|
|
target = 'reserve';
|
|
} else {
|
|
next.primaryFailedSince = null;
|
|
if (input.primaryHealth !== 'healthy') {
|
|
next.primaryRecoveredSince = null;
|
|
next.quietSince = null;
|
|
return { status: 'reserve', reason: 'primary-not-recovered', switchTo: null, memory: next, nextDecisionAt: null };
|
|
}
|
|
if ((input.primaryQuarantineUntil || 0) > input.now) {
|
|
next.primaryRecoveredSince = null;
|
|
next.quietSince = null;
|
|
return {
|
|
status: 'reserve',
|
|
reason: 'recovery-hold',
|
|
switchTo: null,
|
|
memory: next,
|
|
nextDecisionAt: input.primaryQuarantineUntil || null,
|
|
};
|
|
}
|
|
next.primaryRecoveredSince ??= input.now;
|
|
const recoveredAt = next.primaryRecoveredSince + input.policy.recoveryWindowMs;
|
|
readyAt = input.reserveHealth === 'unhealthy'
|
|
? recoveredAt
|
|
: Math.max(recoveredAt, input.holdUntil || 0, input.primaryQuarantineUntil || 0);
|
|
if (input.now < readyAt) {
|
|
next.quietSince = null;
|
|
return { status: 'reserve', reason: 'recovery-hold', switchTo: null, memory: next, nextDecisionAt: readyAt };
|
|
}
|
|
target = 'primary';
|
|
}
|
|
|
|
if (input.policy.trafficGuard.enabled) {
|
|
if (input.activity === 'unknown') {
|
|
next.quietSince = null;
|
|
return { status: 'blocked', reason: 'activity-unknown', switchTo: null, memory: next, nextDecisionAt: null };
|
|
}
|
|
if (input.activity === 'active') {
|
|
next.quietSince = null;
|
|
return { status: 'waiting-for-idle', reason: 'active-traffic', switchTo: null, memory: next, nextDecisionAt: null };
|
|
}
|
|
next.quietSince ??= input.now;
|
|
readyAt = next.quietSince + input.policy.trafficGuard.quietWindowMs;
|
|
if (input.now < readyAt) {
|
|
return { status: 'waiting-for-idle', reason: 'quiet-window', switchTo: null, memory: next, nextDecisionAt: readyAt };
|
|
}
|
|
}
|
|
return { status: 'switching', reason: target === 'reserve' ? 'primary-failed' : 'primary-recovered', switchTo: target, memory: next, nextDecisionAt: null };
|
|
}
|