Harden traffic history worker lifecycle and query performance

2026-09-19 09:58:54 +03:00
parent 74c5b66482
commit 4c58384056
14 changed files with 509 additions and 89 deletions
@@ -0,0 +1,65 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { performance } from 'node:perf_hooks';
+import test from 'node:test';
+import { openTrafficHistoryStore } from '../../dist/server/services/trafficHistoryStore.js';
+import { createTrafficHistoryService } from '../../dist/server/services/trafficHistoryService.js';
+import { parseTrafficHistoryQuery } from '../../dist/shared/trafficHistory.js';
+
+test('large history preserves totals and serves concurrent readers through cleanup and filters', async (t) => {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-history-load-'));
+  const file = path.join(directory, 'traffic.sqlite');
+  const at = Math.floor(Date.now() / 60_000) * 60_000;
+  const minutes = process.env.HARBOR_HISTORY_LOAD === '1' ? 3000 : 300;
+  let service;
+  t.after(async () => {
+    await service?.close();
+    fs.rmSync(directory, { recursive: true, force: true });
+  });
+  openTrafficHistoryStore(file).close();
+  const db = new DatabaseSync(file);
+  try {
+    db.exec(`BEGIN;
+      WITH RECURSIVE n(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM n WHERE i<2000)
+      INSERT INTO dimensions SELECT i,'dimension-'||i,'device-'||(i%20),'Device '||(i%20),
+        '192.0.2.1','tproxy-in','service-'||(i%250),'domain-'||(i%250)||'.test',
+        'host-'||i||'.test','203.0.113.'||(i%250),'vpn','vpn-one' FROM n;`);
+    db.prepare(`WITH RECURSIVE minutes(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM minutes WHERE i<?)
+      INSERT INTO buckets SELECT ?-i*60000,60000,d.id,100,200 FROM minutes
+      CROSS JOIN dimensions d WHERE d.id<=1000`).run(minutes, at);
+    db.prepare('INSERT INTO buckets SELECT ?,60000,id,100,200 FROM dimensions WHERE id>1000').run(at - 91 * 86_400_000);
+    db.exec('COMMIT');
+    // Exercise the additive index upgrade on a populated pre-fix database.
+    db.exec('DROP INDEX buckets_dimension_time');
+  } finally { db.close(); }
+
+  service = createTrafficHistoryService({ filePath: file, source: () => 'live' });
+  const query = parseTrafficHistoryQuery(new URLSearchParams(`range=7d&until=${at}`));
+  const start = performance.now();
+  const concurrent = await Promise.all(Array.from({ length: 4 }, () => service.query(query)));
+  const concurrentMs = performance.now() - start;
+  for (const result of concurrent) {
+    assert.equal(result.storage.status, 'ready');
+    assert.equal(result.totals.downloadBytes, String(minutes * 1000 * 200));
+    assert.equal(result.origins.length, 20);
+    assert.equal(result.nextOffset, 100);
+  }
+  const timings = [];
+  for (const [filter, dimensions] of [
+    [{ originId: 'device-1' }, 50],
+    [{ search: 'host-99' }, 11],
+    [{ level: 'ip', service: 'service-1', domain: 'domain-1.test', hostname: 'host-1.test' }, 1],
+    [{ offset: 300 }, 1000],
+  ]) {
+    const begin = performance.now();
+    const result = await service.query({ ...query, ...filter });
+    timings.push({ filter, ms: performance.now() - begin });
+    assert.equal(result.storage.status, 'ready');
+    assert.equal(result.totals.downloadBytes, String(minutes * dimensions * 200));
+    if (filter.offset) assert.equal(result.rows.length, 0);
+  }
+  t.diagnostic(JSON.stringify({ buckets: minutes * 1000, concurrentMs, timings }));
+});