Update Harbor client and gateway integration workflows

2026-08-19 18:16:10 +03:00
parent 416b2b294a
commit daec12e013
63 changed files with 5016 additions and 108 deletions
@@ -0,0 +1,671 @@
+import crypto from 'node:crypto';
+import {
+  createIdleFailoverSnapshot,
+  isFailoverConfigured,
+  nextFailoverDecision,
+  normalizeFailoverPolicy,
+  type AppliedFailoverPolicy,
+  type FailoverDecisionMemory,
+  type FailoverHealth,
+  type FailoverPolicy,
+  type FailoverRole,
+  type FailoverSnapshot,
+} from '../../../shared/failover.js';
+import type { HarborServer, StoredState } from '../../../shared/contracts/state.js';
+import type { ActivityJournalEventInput } from '../../../shared/activityJournal.js';
+import { HarborError } from '../../../shared/errors.js';
+
+interface Candidate {
+  config: unknown;
+  applied: AppliedFailoverPolicy;
+}
+
+interface FailoverServiceDependencies {
+  state: {
+    read(): StoredState;
+    update(mutator: (state: StoredState) => Record<string, unknown>): StoredState;
+  };
+  runtime: { isRunning(): Promise<boolean> };
+  dataplane: {
+    checkConfig(config: unknown): Promise<unknown>;
+    runFailoverProbe(role: FailoverRole, services: unknown, target: string, timeoutMs: number): Promise<unknown>;
+    readFailoverSelector(): Promise<unknown>;
+    selectFailoverRole(role: FailoverRole): Promise<unknown>;
+    setFailoverActivityEnabled(enabled: boolean): Promise<unknown>;
+    readFailoverActivity(thresholdBytesPerSecond: number): Promise<unknown>;
+  };
+  buildCandidate(state: StoredState): Candidate;
+  serialize<T>(operation: () => Promise<T>): Promise<T>;
+  scheduler?: {
+    setTimeout(callback: () => void, intervalMs: number): NodeJS.Timeout;
+    clearTimeout(timer: NodeJS.Timeout): void;
+  };
+  now?: () => Date;
+  onWarning?: (error: unknown) => void;
+  onSwitch?: (from: FailoverRole, to: FailoverRole, reason: string) => void;
+  onEvent?: (event: ActivityJournalEventInput) => void;
+}
+
+const record = (value: unknown): Record<string, unknown> => (
+  value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : {}
+);
+
+function targetServer(state: StoredState, role: FailoverRole): HarborServer | null {
+  const target = state.appliedFailoverPolicy?.[role] || state.failoverPolicy[role];
+  return state.profiles.find(({ id }) => id === target.profileId)
+    ?.servers.find(({ id }) => id === target.serverId) || null;
+}
+
+function currentRole(state: StoredState): FailoverRole | null {
+  const applied = state.appliedFailoverPolicy;
+  if (!applied) return null;
+  for (const role of ['primary', 'reserve'] as const) {
+    if (
+      state.appliedProfileId === applied[role].profileId
+      && state.appliedServerId === applied[role].serverId
+    ) return role;
+  }
+  return null;
+}
+
+function probeHealth(value: unknown): boolean {
+  const vpn = record(record(value).vpn);
+  const sites = Array.isArray(vpn.sites) ? vpn.sites.map(record) : [];
+  return sites.length === 1 && sites[0].status === 'available';
+}
+
+function safeErrorCode(error: unknown) {
+  const code = error && typeof error === 'object' && 'code' in error ? String(error.code) : '';
+  return /^[A-Z0-9_]{1,50}$/.test(code) ? code : 'UNKNOWN';
+}
+
+export function createFailoverService(dependencies: FailoverServiceDependencies) {
+  const scheduler = dependencies.scheduler || {
+    setTimeout: (callback: () => void, intervalMs: number) => setTimeout(callback, intervalMs),
+    clearTimeout: (timer: NodeJS.Timeout) => clearTimeout(timer),
+  };
+  const now = dependencies.now || (() => new Date());
+  const epoch = crypto.randomUUID();
+  let sequence = 0;
+  let generation = 0;
+  let timer: NodeJS.Timeout | null = null;
+  let collectorEnabled: boolean | null = null;
+  let roundPromise: Promise<void> | null = null;
+  let decisionMemory: FailoverDecisionMemory | undefined;
+  let snapshot = createIdleFailoverSnapshot(dependencies.state.read().failoverPolicy, epoch, sequence);
+
+  function appliedMatchesDesired(state: StoredState) {
+    if (!state.appliedFailoverPolicy) return false;
+    try {
+      return JSON.stringify(dependencies.buildCandidate(state).applied)
+        === JSON.stringify(state.appliedFailoverPolicy);
+    } catch {
+      return false;
+    }
+  }
+
+  function publish(next: FailoverSnapshot) {
+    sequence += 1;
+    snapshot = { ...next, observationEpoch: epoch, observationSequence: sequence };
+  }
+
+  function clearTimer() {
+    if (timer) scheduler.clearTimeout(timer);
+    timer = null;
+  }
+
+  function schedule(delay: number) {
+    clearTimer();
+    timer = scheduler.setTimeout(() => {
+      timer = null;
+      void runRound().catch(dependencies.onWarning);
+    }, delay);
+    timer.unref?.();
+  }
+
+  async function disableCollector() {
+    if (collectorEnabled === false) return;
+    await dependencies.dataplane.setFailoverActivityEnabled(false);
+    collectorEnabled = false;
+  }
+
+  async function deactivate(policy: FailoverPolicy, passiveRole: FailoverRole | null = null) {
+    generation += 1;
+    clearTimer();
+    decisionMemory = undefined;
+    await disableCollector();
+    const idle = createIdleFailoverSnapshot(policy, epoch, sequence);
+    if (passiveRole) {
+      idle.activation = 'passive-loaded';
+      idle.currentRole = passiveRole;
+      idle.reason = 'disabled';
+    }
+    publish(idle);
+  }
+
+  function activeSnapshot(state: StoredState, status: FailoverSnapshot['status'] = 'observing'): FailoverSnapshot {
+    const role = state.failoverRuntimeState.reasonCode === 'selector-unknown' ? null : currentRole(state);
+    const channel = (target: typeof state.failoverPolicy.primary) => ({
+      target,
+      health: 'unknown' as FailoverHealth,
+      failingServiceIds: [],
+      checkedAt: null,
+      stateSince: null,
+    });
+    return {
+      observationEpoch: epoch,
+      observationSequence: sequence,
+      configured: isFailoverConfigured(state.failoverPolicy),
+      enabled: state.failoverPolicy.enabled,
+      paused: state.failoverPolicy.paused,
+      activation: appliedMatchesDesired(state) ? 'active' : 'pending',
+      currentRole: role || 'other',
+      status,
+      primary: channel(state.appliedFailoverPolicy?.primary || state.failoverPolicy.primary),
+      reserve: channel(state.appliedFailoverPolicy?.reserve || state.failoverPolicy.reserve),
+      nextDecisionAt: null,
+      reason: null,
+      trafficActivity: null,
+      policy: state.failoverPolicy,
+    };
+  }
+
+  async function reconcile() {
+    let state = dependencies.state.read();
+    const policy = state.failoverPolicy;
+    if (!policy.enabled) {
+      const role = currentRole(state);
+      const passiveRole = role && await dependencies.runtime.isRunning() ? role : null;
+      return deactivate(policy, passiveRole);
+    }
+    const running = await dependencies.runtime.isRunning();
+    if (!running || !state.appliedFailoverPolicy || !currentRole(state)) {
+      generation += 1;
+      clearTimer();
+      await disableCollector();
+      const pending = activeSnapshot(state, 'idle');
+      pending.activation = running ? 'pending' : 'inactive';
+      pending.reason = running ? 'pending-activation' : 'vpn-stopped';
+      publish(pending);
+      return;
+    }
+    const role = currentRole(state)!;
+    const selected = record(await dependencies.dataplane.readFailoverSelector());
+    if (selected.role !== role) await dependencies.dataplane.selectFailoverRole(role);
+    if (state.failoverRuntimeState.reasonCode === 'selector-unknown') {
+      state = dependencies.state.update((current) => ({
+        ...current,
+        failoverRuntimeState: { ...current.failoverRuntimeState, reasonCode: null },
+      }));
+    }
+    if (collectorEnabled !== true) {
+      await dependencies.dataplane.setFailoverActivityEnabled(true);
+      collectorEnabled = true;
+    }
+    generation += 1;
+    const active = activeSnapshot(state);
+    if (active.activation === 'pending') active.reason = 'pending-activation';
+    publish(active);
+    schedule(0);
+  }
+
+  async function reconcileAfterCommit() {
+    try {
+      await reconcile();
+    } catch (error) {
+      dependencies.onWarning?.(error);
+      const failed = activeSnapshot(dependencies.state.read(), 'error');
+      failed.reason = 'reconcile-failed';
+      publish(failed);
+    }
+  }
+
+  async function assessRole(role: FailoverRole, policy: FailoverPolicy) {
+    const custom = dependencies.state.read().diagnostics.customServices;
+    const results = await Promise.all(policy.checks.map(async (check) => {
+      try {
+        return {
+          id: check.serviceId,
+          ok: probeHealth(await dependencies.dataplane.runFailoverProbe(
+            role,
+            custom,
+            `site:${check.serviceId}`,
+            check.timeoutMs,
+          )),
+        };
+      } catch {
+        return { id: check.serviceId, ok: null };
+      }
+    }));
+    return {
+      health: results.some(({ ok }) => ok === null)
+        ? 'unknown' as const
+        : results.every(({ ok }) => ok) ? 'healthy' as const : 'unhealthy' as const,
+      failingServiceIds: results.filter(({ ok }) => ok === false).map(({ id }) => id),
+    };
+  }
+
+  async function switchWithinQueue(role: FailoverRole, reason: string) {
+      const before = dependencies.state.read();
+      const from = currentRole(before);
+      if (!from || from === role) return;
+      try {
+        await dependencies.dataplane.selectFailoverRole(role);
+        const server = targetServer(before, role);
+        if (!server) throw new HarborError('SERVER_NOT_FOUND');
+        const target = before.appliedFailoverPolicy![role];
+        const switchedAt = now().toISOString();
+        const cutoff = now().getTime() - before.failoverPolicy.flapProtection.windowMs;
+        const history = role === 'reserve'
+          ? [...before.failoverRuntimeState.failoverHistory.filter((value) => Date.parse(value) >= cutoff), switchedAt]
+          : before.failoverRuntimeState.failoverHistory.filter((value) => Date.parse(value) >= cutoff);
+        const quarantine = history.length >= before.failoverPolicy.flapProtection.count
+          ? new Date(now().getTime() + before.failoverPolicy.flapProtection.quarantineMs).toISOString()
+          : before.failoverRuntimeState.primaryQuarantineUntil;
+        dependencies.state.update((state) => ({
+          ...state,
+          appliedProfileId: target.profileId,
+          appliedServerId: target.serverId,
+          appliedServerSnapshot: server,
+          failoverPolicy: reason === 'manual'
+            ? { ...state.failoverPolicy, paused: true }
+            : role === 'primary' ? { ...state.failoverPolicy, paused: false } : state.failoverPolicy,
+          failoverRuntimeState: {
+            ...state.failoverRuntimeState,
+            lastSwitchAt: switchedAt,
+            holdUntil: role === 'reserve'
+              ? new Date(now().getTime() + state.failoverPolicy.minimumReserveMs).toISOString()
+              : null,
+            primaryQuarantineUntil: quarantine,
+            failoverHistory: history,
+            reasonCode: reason,
+          },
+        }));
+      } catch (error) {
+        try {
+          await dependencies.dataplane.selectFailoverRole(from);
+        } catch (rollback) {
+          generation += 1;
+          clearTimer();
+          decisionMemory = undefined;
+          dependencies.state.update((state) => ({
+            ...state,
+            failoverPolicy: { ...state.failoverPolicy, paused: true },
+            failoverRuntimeState: { ...state.failoverRuntimeState, reasonCode: 'selector-unknown' },
+          }));
+          const failed = activeSnapshot(dependencies.state.read(), 'error');
+          failed.reason = 'selector-unknown';
+          publish(failed);
+          throw new AggregateError([error, rollback], 'Failover selector rollback failed');
+        }
+        throw error;
+      }
+      dependencies.onSwitch?.(from, role, reason);
+      dependencies.onEvent?.({
+        type: 'failover.switched',
+        severity: 'info',
+        source: 'failover',
+        dedupeKey: `failover.switched:${dependencies.state.read().revision}`,
+        data: {
+          fromRole: from,
+          toRole: role,
+          primaryLabel: targetServer(dependencies.state.read(), 'primary')?.label || 'Primary',
+          reserveLabel: targetServer(dependencies.state.read(), 'reserve')?.label || 'Reserve',
+          reason,
+          manual: reason === 'manual',
+        },
+      });
+  }
+
+  async function performRound(capturedGeneration: number) {
+    const prepared = await dependencies.serialize(async () => {
+      const state = dependencies.state.read();
+      const role = currentRole(state);
+      if (
+        capturedGeneration !== generation
+        || !state.failoverPolicy.enabled
+        || !state.appliedFailoverPolicy
+        || !role
+      ) return null;
+      const selected = record(await dependencies.dataplane.readFailoverSelector());
+      if (selected.role !== role) await dependencies.dataplane.selectFailoverRole(role);
+      await dependencies.dataplane.setFailoverActivityEnabled(true);
+      collectorEnabled = true;
+      const latest = dependencies.state.read();
+      return capturedGeneration === generation
+        && latest.failoverPolicy.enabled
+        && currentRole(latest) === role
+        ? { state: latest, policy: latest.failoverPolicy, role }
+        : null;
+    });
+    if (!prepared) return;
+    const { state, policy, role } = prepared;
+    const checkedAt = now().toISOString();
+    let primary;
+    let reserve;
+    try {
+      [primary, reserve] = await Promise.all([
+        assessRole('primary', policy),
+        assessRole('reserve', policy),
+      ]);
+    } catch {
+      primary = { health: 'unknown' as const, failingServiceIds: [] };
+      reserve = { health: 'unknown' as const, failingServiceIds: [] };
+    }
+    if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return;
+    const activityResponse = record(await dependencies.dataplane.readFailoverActivity(
+      policy.trafficGuard.thresholdBytesPerSecond,
+    ));
+    if (capturedGeneration !== generation || !dependencies.state.read().failoverPolicy.enabled) return;
+    const activity = record(activityResponse.activity);
+    const observedAt = typeof activity.observedAt === 'string' ? Date.parse(activity.observedAt) : NaN;
+    const activityState = Number.isFinite(observedAt) && now().getTime() - observedAt <= 4_000
+      && (activity.state === 'active' || activity.state === 'quiet')
+      ? activity.state
+      : 'unknown';
+    const decision = nextFailoverDecision({
+      now: now().getTime(),
+      policy,
+      currentRole: role,
+      primaryHealth: primary.health,
+      reserveHealth: reserve.health,
+      activity: activityState,
+      holdUntil: Date.parse(state.failoverRuntimeState.holdUntil || '') || null,
+      primaryQuarantineUntil: Date.parse(state.failoverRuntimeState.primaryQuarantineUntil || '') || null,
+      memory: decisionMemory,
+    });
+    decisionMemory = decision.memory;
+    const previousSnapshot = snapshot;
+    const next = activeSnapshot(state, decision.status);
+    next.currentRole = role;
+    next.primary = {
+      ...next.primary,
+      ...primary,
+      checkedAt,
+      stateSince: previousSnapshot.primary.health === primary.health
+        ? previousSnapshot.primary.stateSince || checkedAt
+        : checkedAt,
+    };
+    next.reserve = {
+      ...next.reserve,
+      ...reserve,
+      checkedAt,
+      stateSince: previousSnapshot.reserve.health === reserve.health
+        ? previousSnapshot.reserve.stateSince || checkedAt
+        : checkedAt,
+    };
+    next.reason = decision.reason;
+    next.nextDecisionAt = decision.nextDecisionAt ? new Date(decision.nextDecisionAt).toISOString() : null;
+    next.trafficActivity = activityState === 'unknown' ? {
+      state: 'unknown',
+      observedAt: Number.isFinite(observedAt) ? new Date(observedAt).toISOString() : checkedAt,
+      windowMs: 10_000,
+      thresholdBytesPerSecond: policy.trafficGuard.thresholdBytesPerSecond,
+      totalBytesPerSecond: 0,
+      transmittingConnections: 0,
+      quietSince: null,
+      switchTarget: decision.switchTo,
+      blockers: [],
+    } : {
+      state: activityState,
+      observedAt: String(activity.observedAt),
+      windowMs: Number(activity.windowMs) || 10_000,
+      thresholdBytesPerSecond: policy.trafficGuard.thresholdBytesPerSecond,
+      totalBytesPerSecond: Number(activity.totalBytesPerSecond) || 0,
+      transmittingConnections: Number(activity.transmittingConnections) || 0,
+      quietSince: typeof activity.quietSince === 'string' ? activity.quietSince : null,
+      switchTarget: decision.switchTo,
+      blockers: (Array.isArray(activity.blockers) ? activity.blockers : []).slice(0, 3) as FailoverSnapshot['trafficActivity'] extends infer T ? T extends { blockers: infer B } ? B : never : never,
+    };
+    publish(next);
+    if (decision.status === 'waiting-for-idle' && previousSnapshot.status !== 'waiting-for-idle') {
+      dependencies.onEvent?.({
+        type: 'failover.waiting_for_idle',
+        severity: 'info',
+        source: 'failover',
+        dedupeKey: `failover.waiting_for_idle:${state.revision}:${role}:${decision.reason}`,
+        data: { fromRole: role, toRole: decision.switchTo || (role === 'primary' ? 'reserve' : 'primary'), reason: decision.reason },
+      });
+    }
+    if (decision.reason === 'both-unhealthy' && previousSnapshot.reason !== 'both-unhealthy') {
+      dependencies.onEvent?.({
+        type: 'failover.both_unhealthy',
+        severity: 'warning',
+        source: 'failover',
+        dedupeKey: `failover.both_unhealthy:${state.revision}`,
+        data: { reason: decision.reason },
+      });
+    }
+    if (primary.health === 'healthy' && previousSnapshot.primary.health === 'unhealthy') {
+      dependencies.onEvent?.({
+        type: 'failover.recovered',
+        severity: 'info',
+        source: 'failover',
+        dedupeKey: `failover.recovered:${state.revision}:primary`,
+        data: { role: 'primary', reason: decision.reason },
+      });
+    }
+    if (decision.switchTo) {
+      try {
+        const switched = await dependencies.serialize(async () => {
+          const current = dependencies.state.read();
+          if (
+            capturedGeneration !== generation
+            || !current.failoverPolicy.enabled
+            || current.failoverPolicy.paused
+            || currentRole(current) !== role
+            || !current.appliedFailoverPolicy
+          ) return false;
+          let freshPrimary;
+          let freshReserve;
+          try {
+            [freshPrimary, freshReserve] = await Promise.all([
+              assessRole('primary', current.failoverPolicy),
+              assessRole('reserve', current.failoverPolicy),
+            ]);
+          } catch {
+            return false;
+          }
+          const healthStillAllowsSwitch = decision.switchTo === 'reserve'
+            ? freshPrimary.health === 'unhealthy' && freshReserve.health === 'healthy'
+            : freshPrimary.health === 'healthy';
+          if (!healthStillAllowsSwitch || capturedGeneration !== generation) return false;
+          if (current.failoverPolicy.trafficGuard.enabled) {
+            const freshResponse = record(await dependencies.dataplane.readFailoverActivity(
+              current.failoverPolicy.trafficGuard.thresholdBytesPerSecond,
+            ));
+            const freshActivity = record(freshResponse.activity);
+            const freshObservedAt = typeof freshActivity.observedAt === 'string'
+              ? Date.parse(freshActivity.observedAt)
+              : NaN;
+            const freshQuietSince = typeof freshActivity.quietSince === 'string'
+              ? Date.parse(freshActivity.quietSince)
+              : NaN;
+            if (
+              capturedGeneration !== generation
+              || freshActivity.state !== 'quiet'
+              || !Number.isFinite(freshObservedAt)
+              || now().getTime() - freshObservedAt > 4_000
+              || !Number.isFinite(freshQuietSince)
+              || now().getTime() - freshQuietSince < current.failoverPolicy.trafficGuard.quietWindowMs
+            ) return false;
+          }
+          await switchWithinQueue(decision.switchTo!, decision.reason);
+          return true;
+        });
+        if (!switched) {
+          const cancelled = activeSnapshot(dependencies.state.read(), 'observing');
+          cancelled.reason = 'revalidation-required';
+          publish(cancelled);
+          decisionMemory = undefined;
+          return;
+        }
+      } catch (error) {
+        const failed = activeSnapshot(dependencies.state.read(), 'error');
+        failed.reason = dependencies.state.read().failoverRuntimeState.reasonCode === 'selector-unknown'
+          ? 'selector-unknown'
+          : 'switch-failed';
+        publish(failed);
+        dependencies.onEvent?.({
+          type: 'failover.switch_failed',
+          severity: 'error',
+          source: 'failover',
+          dedupeKey: `failover.switch_failed:${state.revision}:${role}:${decision.switchTo}`,
+          data: { fromRole: role, toRole: decision.switchTo, reason: decision.reason, errorCode: safeErrorCode(error) },
+        });
+        throw error;
+      }
+      if (capturedGeneration !== generation) return;
+      publish(activeSnapshot(dependencies.state.read(), decision.switchTo === 'reserve' ? 'reserve' : 'primary'));
+      decisionMemory = undefined;
+    }
+  }
+
+  async function runRound() {
+    if (roundPromise) return roundPromise;
+    const capturedGeneration = generation;
+    roundPromise = performRound(capturedGeneration).finally(() => {
+      roundPromise = null;
+      const policy = dependencies.state.read().failoverPolicy;
+      if (capturedGeneration === generation && policy.enabled && dependencies.state.read().appliedFailoverPolicy) {
+        const decisionAt = snapshot.nextDecisionAt ? Date.parse(snapshot.nextDecisionAt) : NaN;
+        const decisionDelay = Number.isFinite(decisionAt)
+          ? Math.max(250, decisionAt - now().getTime())
+          : policy.intervalMs;
+        schedule(Math.min(policy.intervalMs, decisionDelay));
+      }
+    });
+    return roundPromise;
+  }
+
+  function save(value: unknown) {
+    return dependencies.serialize(async () => {
+    let policy: FailoverPolicy;
+    try {
+      policy = normalizeFailoverPolicy(value, { strict: true });
+    } catch (cause) {
+      throw new HarborError('REQUEST_INVALID', { cause });
+    }
+    if (policy.enabled && !isFailoverConfigured(policy)) throw new HarborError('REQUEST_INVALID');
+    const before = dependencies.state.read();
+    const candidateState = { ...before, failoverPolicy: policy };
+    if (policy.enabled) {
+      const candidate = dependencies.buildCandidate(candidateState);
+      await dependencies.dataplane.checkConfig(candidate.config);
+    }
+    dependencies.state.update((state) => {
+      const role = before.failoverPolicy.enabled && !policy.enabled ? currentRole(state) : null;
+      const target = role ? state.appliedFailoverPolicy?.[role] : null;
+      return {
+        ...state,
+        failoverPolicy: policy,
+        ...(target ? {
+          desiredProfileId: target.profileId,
+          profiles: state.profiles.map((profile) => profile.id === target.profileId
+            ? { ...profile, desiredServerId: target.serverId }
+            : profile),
+        } : {}),
+      };
+    });
+    decisionMemory = undefined;
+    if (before.failoverPolicy.enabled !== policy.enabled) {
+      dependencies.onEvent?.({
+        type: policy.enabled ? 'failover.enabled' : 'failover.disabled',
+        severity: 'info',
+        source: 'failover',
+        dedupeKey: `failover.${policy.enabled ? 'enabled' : 'disabled'}:${dependencies.state.read().revision}`,
+        data: {},
+      });
+    }
+    await reconcileAfterCommit();
+    });
+  }
+
+  function pause(paused: boolean) {
+    return dependencies.serialize(async () => {
+    const before = dependencies.state.read();
+    if (!paused && (
+      !before.appliedFailoverPolicy
+      || !targetServer(before, 'primary')
+      || !targetServer(before, 'reserve')
+    )) throw new HarborError('REQUEST_INVALID');
+    dependencies.state.update((state) => ({
+      ...state,
+      failoverPolicy: { ...state.failoverPolicy, paused },
+    }));
+    decisionMemory = undefined;
+    dependencies.onEvent?.({
+      type: paused ? 'failover.paused' : 'failover.resumed',
+      severity: 'info',
+      source: 'failover',
+      dedupeKey: `failover.${paused ? 'paused' : 'resumed'}:${dependencies.state.read().revision}`,
+      data: {},
+    });
+    await reconcileAfterCommit();
+    });
+  }
+
+  async function manualSwitch(role: FailoverRole) {
+    await dependencies.serialize(async () => {
+      const state = dependencies.state.read();
+      if (!state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !currentRole(state)) {
+        throw new HarborError('REQUEST_INVALID');
+      }
+      await switchWithinQueue(role, 'manual');
+    });
+    await reconcileAfterCommit();
+  }
+
+  const prepareActivation = (role: FailoverRole) => dependencies.dataplane.selectFailoverRole(role);
+
+  async function restoreAppliedActivation(state: StoredState) {
+    if (!state.appliedFailoverPolicy) return;
+    const role = currentRole(state);
+    if (!role) throw new HarborError('CONFIG_INVALID');
+    await prepareActivation(role);
+  }
+
+  function checkNow() {
+    return dependencies.serialize(async () => {
+      const state = dependencies.state.read();
+      if (!state.failoverPolicy.enabled || !state.appliedFailoverPolicy || !currentRole(state)) {
+        throw new HarborError('REQUEST_INVALID');
+      }
+      generation += 1;
+      clearTimer();
+      const checkedAt = now().toISOString();
+      let primary;
+      let reserve;
+      try {
+        [primary, reserve] = await Promise.all([
+          assessRole('primary', state.failoverPolicy),
+          assessRole('reserve', state.failoverPolicy),
+        ]);
+      } catch {
+        primary = { health: 'unknown' as const, failingServiceIds: [] };
+        reserve = { health: 'unknown' as const, failingServiceIds: [] };
+      }
+      const next = activeSnapshot(dependencies.state.read(), 'observing');
+      next.primary = { ...next.primary, ...primary, checkedAt, stateSince: checkedAt };
+      next.reserve = { ...next.reserve, ...reserve, checkedAt, stateSince: checkedAt };
+      next.reason = 'manual-check';
+      publish(next);
+      schedule(state.failoverPolicy.intervalMs);
+    });
+  }
+
+  return {
+    snapshot: () => snapshot,
+    save,
+    pause,
+    manualSwitch,
+    checkNow,
+    prepareActivation,
+    restoreAppliedActivation,
+    reconcile,
+    runRound,
+    shutdown: () => deactivate(dependencies.state.read().failoverPolicy),
+  };
+}
+
+export type FailoverService = ReturnType<typeof createFailoverService>;