Add failover channel events to activity journal

test/server/failover-service.test.js

@@ -282,7 +282,9 @@ test('runtime rollback restores the role from applied truth even while disabled'
 
 test('an in-flight observation is discarded after failover is disabled', async () => {
   const pending = [];
+  const events = [];
   const testHarness = harness(serviceState(), {
+    onEvent: (event) => events.push(event),
     dataplane: {
       checkConfig: async () => ({}),
       runFailoverProbe: (role) => new Promise((resolve) => pending.push({ role, resolve })),
@@ -302,6 +304,7 @@ test('an in-flight observation is discarded after failover is disabled', async (
   assert.equal(testHarness.read().failoverPolicy.enabled, false);
   assert.equal(testHarness.calls.some((call) => call.startsWith('select:reserve')), false);
   assert.deepEqual(testHarness.calls.filter((call) => call.startsWith('activity:')), ['activity:true', 'activity:true', 'activity:false']);
+  assert.deepEqual(events.filter(({ type }) => /_(?:unavailable|recovered)$/.test(type)), []);
 });
 
 test('one channel probe failure does not erase the other channel health', async () => {
@@ -595,7 +598,7 @@ test('disabling on reserve preserves it as the next ordinary single-channel targ
   assert.equal(testHarness.service.snapshot().activation, 'passive-loaded');
 });
 
-test('significant health states append once without logging every observation', async () => {
+test('background health transitions cover both channels without duplicate observations', async () => {
   let clock = 0;
   const health = { primary: 'unavailable', reserve: 'available' };
   const events = [];
@@ -611,23 +614,153 @@ test('significant health states append once without logging every observation',
       readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date(clock).toISOString() } }),
     },
   });
+  const healthEvents = () => events.filter(({ type }) => /_(?:unavailable|recovered)$/.test(type));
+  const round = async () => {
+    await testHarness.service.runRound();
+    clock += 1_000;
+  };
+
   await testHarness.service.reconcile();
+  await round();
+  await round();
+  health.reserve = 'unavailable';
+  await round();
+  await round();
+  health.primary = 'available';
+  await round();
+  health.primary = 'unavailable';
+  await round();
+  health.reserve = 'available';
+  await round();
+  health.reserve = 'unavailable';
+  await round();
+
+  assert.deepEqual(healthEvents().map(({ type, severity, data }) => ({ type, severity, role: data.role })), [
+    { type: 'failover.primary_unavailable', severity: 'warning', role: 'primary' },
+    { type: 'failover.reserve_unavailable', severity: 'warning', role: 'reserve' },
+    { type: 'failover.primary_recovered', severity: 'info', role: 'primary' },
+    { type: 'failover.primary_unavailable', severity: 'warning', role: 'primary' },
+    { type: 'failover.reserve_recovered', severity: 'info', role: 'reserve' },
+    { type: 'failover.reserve_unavailable', severity: 'warning', role: 'reserve' },
+  ]);
+  assert.ok(healthEvents().every(({ dedupeKey }) => dedupeKey === null));
+  assert.equal(events.some(({ type }) => type === 'failover.recovered'), false);
+  assert.equal(events.filter(({ type }) => type === 'failover.both_unhealthy').length, 3);
+});
+
+test('activity read failure does not hide completed channel health transitions', async () => {
+  const events = [];
+  const testHarness = harness(serviceState(), {
+    onEvent: (event) => events.push(event),
+    dataplane: {
+      checkConfig: async () => ({}),
+      runFailoverProbe: async (role) => ({
+        vpn: { sites: [{ status: role === 'primary' ? 'unavailable' : 'available' }] },
+      }),
+      readFailoverSelector: async () => ({ role: 'primary' }),
+      selectFailoverRole: async (role) => ({ role }),
+      setFailoverActivityEnabled: async () => {},
+      readFailoverActivity: async () => { throw new Error('activity unavailable'); },
+    },
+  });
+
+  await testHarness.service.reconcile();
+  await assert.rejects(testHarness.service.runRound(), /activity unavailable/);
+  assert.deepEqual(events.map(({ type }) => type), ['failover.primary_unavailable']);
+  assert.equal(events.some(({ type }) => ['failover.waiting_for_idle', 'failover.both_unhealthy', 'failover.switched', 'failover.switch_failed'].includes(type)), false);
+});
+
+test('manual check does not strand monitoring behind a stale background round', async () => {
+  const pending = [];
+  const scheduled = [];
+  let holdBackground = true;
+  let probeCalls = 0;
+  const testHarness = harness(serviceState(), {
+    scheduler: {
+      setTimeout: (callback) => {
+        scheduled.push(callback);
+        return { unref() {} };
+      },
+      clearTimeout: () => {},
+    },
+    dataplane: {
+      checkConfig: async () => ({}),
+      runFailoverProbe: (role) => {
+        probeCalls += 1;
+        if (holdBackground) {
+          return new Promise((resolve) => pending.push({ role, resolve }));
+        }
+        return Promise.resolve({ vpn: { sites: [{ status: 'available' }] } });
+      },
+      readFailoverSelector: async () => ({ role: 'primary' }),
+      selectFailoverRole: async (role) => ({ role }),
+      setFailoverActivityEnabled: async () => {},
+      readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
+    },
+  });
+
+  await testHarness.service.reconcile();
+  const oldRound = testHarness.service.runRound();
+  await new Promise(setImmediate);
+  assert.equal(pending.length, 4);
+  holdBackground = false;
+  await testHarness.service.checkNow();
+  assert.equal(probeCalls, 8);
+  scheduled.at(-1)();
+  for (const probe of pending) {
+    probe.resolve({ vpn: { sites: [{ status: 'available' }] } });
+  }
+  await oldRound;
+  await new Promise(setImmediate);
+  await new Promise(setImmediate);
+  assert.equal(probeCalls, 12);
+});
+
+test('manual and background checks share channel transition memory', async () => {
+  const health = { primary: 'available', reserve: 'available' };
+  const events = [];
+  const testHarness = harness(serviceState(), {
+    onEvent: (event) => events.push(event),
+    dataplane: {
+      checkConfig: async () => ({}),
+      runFailoverProbe: async (role) => ({ vpn: { sites: [{ status: health[role] }] } }),
+      readFailoverSelector: async () => ({ role: 'primary' }),
+      selectFailoverRole: async (role) => ({ role }),
+      setFailoverActivityEnabled: async () => {},
+      readFailoverActivity: async () => ({ activity: { state: 'quiet', observedAt: new Date().toISOString() } }),
+    },
+  });
+  const healthTypes = () => events
+    .filter(({ type }) => /_(?:unavailable|recovered)$/.test(type))
+    .map(({ type }) => type);
+
+  await testHarness.service.reconcile();
+  await testHarness.service.checkNow();
+  assert.deepEqual(healthTypes(), []);
+  health.primary = 'unavailable';
+  await testHarness.service.checkNow();
+  assert.deepEqual(healthTypes(), ['failover.primary_unavailable']);
   await testHarness.service.runRound();
-  clock = 30_000;
+  assert.deepEqual(healthTypes(), ['failover.primary_unavailable']);
+  health.primary = 'available';
+  await testHarness.service.checkNow();
+  health.primary = 'unavailable';
   await testHarness.service.runRound();
   health.reserve = 'unavailable';
-  clock = 31_000;
+  await testHarness.service.checkNow();
   await testHarness.service.runRound();
-  clock = 32_000;
-  await testHarness.service.runRound();
-  health.primary = 'available';
-  clock = 33_000;
+  health.reserve = 'available';
+  await testHarness.service.checkNow();
+  health.reserve = 'unavailable';
   await testHarness.service.runRound();
 
-  assert.deepEqual(events.map(({ type }) => type), [
-    'failover.waiting_for_idle',
-    'failover.both_unhealthy',
-    'failover.recovered',
+  assert.deepEqual(healthTypes(), [
+    'failover.primary_unavailable',
+    'failover.primary_recovered',
+    'failover.primary_unavailable',
+    'failover.reserve_unavailable',
+    'failover.reserve_recovered',
+    'failover.reserve_unavailable',
   ]);
 });