Migrate Harbor state and traffic history to SQLite
This commit is contained in:
@@ -5,8 +5,20 @@ import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
import { createActivityJournalService } from '../../dist/server/services/activityJournalService.js';
|
||||
import { openHarborStorage } from '../../dist/server/services/harborStorage.js';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { assertActivityJournalPage } from '../../dist/shared/activityJournal.js';
|
||||
|
||||
function serviceFor(t, filePath, now) {
|
||||
const storage = openHarborStorage(path.dirname(filePath));
|
||||
t.after(() => storage.close());
|
||||
return createActivityJournalService({ db: storage.db, now });
|
||||
}
|
||||
function persisted(filePath) {
|
||||
const db = new DatabaseSync(path.join(path.dirname(filePath), 'harbor.sqlite'), { readOnly: true });
|
||||
try { return { schemaVersion: 1, events: db.prepare('SELECT value FROM journal ORDER BY sequence').all().map((row) => JSON.parse(row.value)) }; }
|
||||
finally { db.close(); }
|
||||
}
|
||||
function fixture(t) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-journal-'));
|
||||
t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
|
||||
@@ -24,7 +36,7 @@ const event = (dedupeKey, profileLabel = 'Home') => ({
|
||||
test('journal appends typed events, deduplicates and keeps stable newest-first cursors', (t) => {
|
||||
let clock = new Date('2026-08-19T10:00:00.000Z');
|
||||
const filePath = fixture(t);
|
||||
const service = createActivityJournalService({ filePath, now: () => clock });
|
||||
const service = serviceFor(t, filePath, () => clock);
|
||||
service.append(event('refresh:1', 'One'));
|
||||
clock = new Date('2026-08-19T10:01:00.000Z');
|
||||
service.append(event('refresh:2', 'Two'));
|
||||
@@ -38,20 +50,20 @@ test('journal appends typed events, deduplicates and keeps stable newest-first c
|
||||
const older = service.page(10, first.nextCursor);
|
||||
assert.deepEqual(older.events.map(({ data }) => data.profileLabel), ['One']);
|
||||
assert.equal(first.events[0].dedupeKey, null);
|
||||
const inode = fs.statSync(filePath).ino;
|
||||
const inode = fs.statSync(path.join(path.dirname(filePath), 'harbor.sqlite')).ino;
|
||||
assert.equal(service.page(10).events.length, 3);
|
||||
assert.equal(fs.statSync(filePath).ino, inode);
|
||||
assert.equal(fs.statSync(path.join(path.dirname(filePath), 'harbor.sqlite')).ino, inode);
|
||||
assert.deepEqual(service.page(10, 'expired-cursor').events, []);
|
||||
});
|
||||
|
||||
test('journal prunes events older than 30 days and rejects unsafe payloads', (t) => {
|
||||
let clock = new Date('2026-07-01T00:00:00.000Z');
|
||||
const filePath = fixture(t);
|
||||
const service = createActivityJournalService({ filePath, now: () => clock });
|
||||
const service = serviceFor(t, filePath, () => clock);
|
||||
service.append(event('old'));
|
||||
clock = new Date('2026-08-19T00:00:00.000Z');
|
||||
assert.deepEqual(service.page().events, []);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(filePath, 'utf8')).events, []);
|
||||
assert.deepEqual(persisted(filePath).events, []);
|
||||
service.append(event('new'));
|
||||
assert.throws(() => service.append({ ...event('unsafe'), data: { rawUrl: 'https://secret' } }), /Unsafe/);
|
||||
service.append({ ...event('ip'), data: { ...event('ip').data, host: '192.168.1.1' } });
|
||||
@@ -68,9 +80,9 @@ test('journal prunes events older than 30 days and rejects unsafe payloads', (t)
|
||||
'subscription.refreshed:user:pass',
|
||||
]) service.append({ ...event('safe'), dedupeKey });
|
||||
service.append({ ...event('safe'), dedupeKey: 'subscription.refreshed:user:pass' });
|
||||
const persisted = fs.readFileSync(filePath, 'utf8');
|
||||
assert.doesNotMatch(persisted, /user:pass|192\.168\.1\.1|192\.168\.1\.7|2001:db8|token=secret/);
|
||||
assert.match(persisted, /subscription\.refreshed:sha256:[a-f0-9]{64}/);
|
||||
const persistedText = JSON.stringify(persisted(filePath));
|
||||
assert.doesNotMatch(persistedText, /user:pass|192\.168\.1\.1|192\.168\.1\.7|2001:db8|token=secret/);
|
||||
assert.match(persistedText, /subscription\.refreshed:sha256:[a-f0-9]{64}/);
|
||||
assert.throws(() => service.append({ ...event('safe'), dedupeKey: 'https://user:pass@example.test/private?token=x' }), /dedupe/i);
|
||||
assert.throws(() => service.append({ ...event('safe'), dedupeKey: 'subscription.refreshed:private/path' }), /dedupe/i);
|
||||
});
|
||||
@@ -84,36 +96,24 @@ test('journal persists the 10,000 event cap when opening an oversized store', (t
|
||||
...event(`event:${index}`),
|
||||
}));
|
||||
fs.writeFileSync(filePath, JSON.stringify({ schemaVersion: 1, events }));
|
||||
const service = createActivityJournalService({
|
||||
filePath,
|
||||
now: () => new Date('2026-08-19T01:00:00.000Z'),
|
||||
});
|
||||
assert.equal(JSON.parse(fs.readFileSync(filePath, 'utf8')).events.length, 10_000);
|
||||
const service = serviceFor(t, filePath, () => new Date('2026-08-19T01:00:00.000Z'));
|
||||
assert.equal(service.page(1).events.length, 1);
|
||||
assert.equal(persisted(filePath).events.length, 10_000);
|
||||
});
|
||||
|
||||
test('corrupt journal is isolated and recovery becomes a safe event', (t) => {
|
||||
test('corrupt journal blocks import and leaves the original intact', (t) => {
|
||||
const filePath = fixture(t);
|
||||
fs.writeFileSync(filePath, '{broken');
|
||||
const service = createActivityJournalService({
|
||||
filePath,
|
||||
now: () => new Date('2026-08-19T12:00:00.000Z'),
|
||||
});
|
||||
const page = service.page();
|
||||
assert.equal(page.storage.status, 'ready');
|
||||
assert.equal(page.events[0].type, 'journal.recovered');
|
||||
assert.ok(fs.readdirSync(path.dirname(filePath)).some((name) => name.includes('.corrupt-')));
|
||||
assert.throws(() => serviceFor(t, filePath), /Cannot migrate activity-journal.json/);
|
||||
assert.equal(fs.readFileSync(filePath, 'utf8'), '{broken');
|
||||
});
|
||||
|
||||
test('journal exposes a latched write failure until a later append succeeds', (t) => {
|
||||
const filePath = fixture(t);
|
||||
const service = createActivityJournalService({ filePath });
|
||||
const service = serviceFor(t, filePath);
|
||||
service.append(event('before-error'));
|
||||
const renameSync = fs.renameSync;
|
||||
fs.renameSync = (source, target) => {
|
||||
if (target === filePath) throw new Error('simulated journal write failure');
|
||||
return renameSync(source, target);
|
||||
};
|
||||
const db = new DatabaseSync(path.join(path.dirname(filePath), 'harbor.sqlite'));
|
||||
db.exec("CREATE TRIGGER reject_journal BEFORE INSERT ON journal BEGIN SELECT RAISE(ABORT, 'simulated journal failure'); END");
|
||||
try {
|
||||
assert.throws(() => service.append(event('lost')));
|
||||
const failed = service.page();
|
||||
@@ -122,7 +122,8 @@ test('journal exposes a latched write failure until a later append succeeds', (t
|
||||
assert.equal(failed.events.length, 1);
|
||||
assert.equal(failed.events[0].dedupeKey, null);
|
||||
} finally {
|
||||
fs.renameSync = renameSync;
|
||||
db.exec('DROP TRIGGER reject_journal');
|
||||
db.close();
|
||||
}
|
||||
service.append(event('recovered'));
|
||||
assert.equal(service.page().storage.status, 'ready');
|
||||
@@ -145,7 +146,7 @@ test('schema version 1 keeps legacy recovery and accepts per-channel health even
|
||||
}],
|
||||
}));
|
||||
let clock = new Date('2026-08-19T10:00:00.000Z');
|
||||
const service = createActivityJournalService({ filePath, now: () => clock });
|
||||
const service = serviceFor(t, filePath, () => clock);
|
||||
const inputs = [
|
||||
['failover.primary_unavailable', 'primary', 'warning', 'probe-failed'],
|
||||
['failover.primary_recovered', 'primary', 'info', 'probe-recovered'],
|
||||
@@ -173,10 +174,10 @@ test('schema version 1 keeps legacy recovery and accepts per-channel health even
|
||||
]);
|
||||
assert.deepEqual(page.events.at(-1).data, { role: 'primary', reason: 'primary-recovered' });
|
||||
assert.equal(page.retentionDays, 30);
|
||||
assert.equal(JSON.parse(fs.readFileSync(filePath, 'utf8')).schemaVersion, 1);
|
||||
assert.equal(persisted(filePath).schemaVersion, 1);
|
||||
});
|
||||
test('journal page parser rejects malformed wire data and strips unknown event fields', (t) => {
|
||||
const service = createActivityJournalService({ filePath: fixture(t) });
|
||||
const service = serviceFor(t, fixture(t));
|
||||
service.append(event('wire'));
|
||||
const page = service.page();
|
||||
const parsed = assertActivityJournalPage({
|
||||
|
||||
@@ -108,6 +108,13 @@ test('compiled dispatcher starts and stops control and dataplane contracts', asy
|
||||
assert.equal(page.status, 200);
|
||||
assert.match(page.type, /^text\/html/);
|
||||
assert.match(page.body, /<div id="root"><\/div>/);
|
||||
const historyResponse = await fetch(`http://127.0.0.1:${port}/api/traffic/history?range=90d`);
|
||||
const history = await historyResponse.json();
|
||||
assert.equal(historyResponse.status, 200);
|
||||
assert.equal(history.storage.status, 'ready');
|
||||
assert.equal(history.period.retentionDays, 90);
|
||||
assert.deepEqual(history.rows, []);
|
||||
assert.equal(fs.existsSync(path.join(controlData, 'harbor.sqlite')), true);
|
||||
await stop(control.child);
|
||||
|
||||
const dataplane = start({
|
||||
@@ -123,6 +130,11 @@ test('compiled dispatcher starts and stops control and dataplane contracts', asy
|
||||
return response.status === 200 ? response.body : null;
|
||||
}, dataplane.child, dataplane.stderr);
|
||||
assert.equal(status.ready, true);
|
||||
const gatewayHistory = await socketRequest(socketPath, '/traffic/history?range=7d');
|
||||
assert.equal(gatewayHistory.status, 200);
|
||||
assert.equal(gatewayHistory.body.storage.status, 'ready');
|
||||
assert.equal(gatewayHistory.body.query.range, '7d');
|
||||
assert.equal(fs.existsSync(path.join(dataplaneData, 'traffic.sqlite')), true);
|
||||
await stop(dataplane.child);
|
||||
});
|
||||
|
||||
@@ -148,7 +160,7 @@ test('production paths use only the compiled dispatcher', () => {
|
||||
assert.doesNotMatch(entrypoint, /\/app\/src/);
|
||||
}
|
||||
assert.match(workflow, /npm run build:production/);
|
||||
assert.match(workflow, /NODE_BUILD_IMAGE: mirror\.gcr\.io\/library\/node:20\.19-bookworm/);
|
||||
assert.match(workflow, /NODE_BUILD_IMAGE: mirror\.gcr\.io\/library\/node:24\.21\.0-bookworm/);
|
||||
assert.match(workflow, /command -v npm[^']+command -v git[^']+test -x \/bin\/bash/);
|
||||
assert.doesNotMatch(workflow, /docker image inspect "\$\{\{ env\.NODE_BUILD_IMAGE \}\}"/);
|
||||
assert.match(legacyBuild, /npm run build:production && docker build/);
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import test from 'node:test';
|
||||
import { openHarborStorage } from '../../dist/server/services/harborStorage.js';
|
||||
import { migrateStoredState } from '../../dist/server/services/stateStore.js';
|
||||
import { migrateDeviceInventoryState } from '../../dist/server/services/deviceInventoryService.js';
|
||||
import { createActivityJournalService } from '../../dist/server/services/activityJournalService.js';
|
||||
|
||||
function fixture(t) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-sqlite-'));
|
||||
const stores = [];
|
||||
t.after(() => {
|
||||
for (const store of stores) if (store.db.isOpen) store.close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
return { directory, open: () => { const store = openHarborStorage(directory); stores.push(store); return store; } };
|
||||
}
|
||||
|
||||
test('atomic import preserves revisions, rules, device checkpoints and journal; JSON becomes backup only', (t) => {
|
||||
const f = fixture(t);
|
||||
const state = migrateStoredState({ revision: 41, routeRules: [{ type: 'domain_suffix', value: 'example.org', enabled: true }] });
|
||||
const mac = 'aa:bb:cc:dd:ee:ff';
|
||||
const devices = migrateDeviceInventoryState({ revision: 17, traffic: {
|
||||
baselinesByMac: { [mac]: { epoch: 'kernel-1', uploadBytes: '9007199254740993', downloadBytes: '123' } },
|
||||
totalsByMac: { [mac]: { uploadBytes: '9007199254740993', downloadBytes: '987', observedAt: '2026-09-10T10:00:00.000Z' } },
|
||||
} });
|
||||
const event = { id: '00000000-0000-4000-8000-000000000001', occurredAt: new Date().toISOString(),
|
||||
type: 'connection.stopped', severity: 'info', source: 'connection', dedupeKey: null, data: {} };
|
||||
for (const [name, value] of [['state.json', state], ['devices.json', devices], ['activity-journal.json', { schemaVersion: 1, events: [event] }]]) {
|
||||
fs.writeFileSync(path.join(f.directory, name), JSON.stringify(value));
|
||||
}
|
||||
const originals = ['state.json', 'devices.json', 'activity-journal.json'].map((name) => fs.readFileSync(path.join(f.directory, name), 'utf8'));
|
||||
let store = f.open();
|
||||
assert.equal(store.imported, true);
|
||||
assert.deepEqual(store.state.read(), state);
|
||||
assert.deepEqual(store.devices.read(), devices);
|
||||
assert.deepEqual(createActivityJournalService({ db: store.db }).page().events, [event]);
|
||||
store.state.update((value) => ({ ...value, revision: 42 }));
|
||||
store.devices.update((value) => ({ ...value, revision: 18 }));
|
||||
store.close();
|
||||
// Even broken obsolete files cannot override or break the canonical SQL state.
|
||||
fs.writeFileSync(path.join(f.directory, 'state.json'), '{obsolete');
|
||||
store = f.open();
|
||||
assert.equal(store.imported, false);
|
||||
assert.equal(store.state.read().revision, 42);
|
||||
assert.equal(store.devices.read().revision, 18);
|
||||
assert.equal(store.devices.read().traffic.baselinesByMac[mac].uploadBytes, '9007199254740993');
|
||||
assert.equal(fs.readFileSync(path.join(f.directory, 'devices.json'), 'utf8'), originals[1]);
|
||||
assert.equal(fs.readFileSync(path.join(f.directory, 'activity-journal.json'), 'utf8'), originals[2]);
|
||||
});
|
||||
|
||||
test('failed import leaves no partial documents and can be retried after fixing the original', (t) => {
|
||||
const f = fixture(t);
|
||||
fs.writeFileSync(path.join(f.directory, 'state.json'), JSON.stringify({ revision: 9 }));
|
||||
fs.writeFileSync(path.join(f.directory, 'devices.json'), '{broken');
|
||||
assert.throws(f.open, /Cannot migrate devices.json/);
|
||||
assert.equal(fs.readFileSync(path.join(f.directory, 'devices.json'), 'utf8'), '{broken');
|
||||
const inspect = new DatabaseSync(path.join(f.directory, 'harbor.sqlite'));
|
||||
assert.equal(inspect.prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE name = 'documents'").get().n, 0);
|
||||
inspect.close();
|
||||
fs.writeFileSync(path.join(f.directory, 'devices.json'), '{}');
|
||||
assert.equal(f.open().state.read().revision, 9);
|
||||
});
|
||||
|
||||
test('an optional legacy null subscription cache migrates without changing the backup', (t) => {
|
||||
const f = fixture(t);
|
||||
fs.writeFileSync(path.join(f.directory, 'state.json'), JSON.stringify({ schemaVersion: 4, revision: 7 }));
|
||||
fs.writeFileSync(path.join(f.directory, 'subscription-cache.json'), 'null');
|
||||
assert.equal(f.open().state.read().revision, 7);
|
||||
assert.equal(fs.readFileSync(path.join(f.directory, 'subscription-cache.json'), 'utf8'), 'null');
|
||||
});
|
||||
|
||||
test('a database constraint failure rolls back the entire import including journal and marker', (t) => {
|
||||
const f = fixture(t);
|
||||
const event = { id: '00000000-0000-4000-8000-000000000001', occurredAt: new Date().toISOString(),
|
||||
type: 'connection.stopped', severity: 'info', source: 'connection', dedupeKey: null, data: {} };
|
||||
fs.writeFileSync(path.join(f.directory, 'activity-journal.json'), JSON.stringify({ schemaVersion: 1, events: [event, event] }));
|
||||
assert.throws(f.open, /UNIQUE/);
|
||||
const db = new DatabaseSync(path.join(f.directory, 'harbor.sqlite'));
|
||||
assert.equal(db.prepare('PRAGMA user_version').get().user_version, 0);
|
||||
assert.equal(db.prepare("SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table'").get().n, 0);
|
||||
db.close();
|
||||
});
|
||||
|
||||
test('durable DB corruption, future versions and invalid synchronous mutations never fall back to old JSON', (t) => {
|
||||
const f = fixture(t);
|
||||
const store = f.open();
|
||||
assert.throws(() => store.state.update(async (state) => state), /synchronous/);
|
||||
assert.equal(store.state.read().revision, 0);
|
||||
store.db.exec('PRAGMA user_version = 99');
|
||||
store.close();
|
||||
fs.writeFileSync(path.join(f.directory, 'state.json'), '{}');
|
||||
assert.throws(f.open, /Unsupported Harbor database/);
|
||||
});
|
||||
@@ -5,6 +5,7 @@ import http from 'node:http';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
import { buildGatewayPresence } from '../../dist/server/gatewayPresence.js';
|
||||
import { normalizeSubscriptionConfig } from '../../dist/server/subscription.js';
|
||||
@@ -151,6 +152,7 @@ async function startClientFixture(t, {
|
||||
hostNetwork,
|
||||
gatewayPresencePort,
|
||||
trafficSource,
|
||||
expectMigrationError,
|
||||
}) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-startup-recovery-'));
|
||||
const { binDirectory, markerPath } = fakeSingbox(directory);
|
||||
@@ -188,6 +190,15 @@ async function startClientFixture(t, {
|
||||
await stopChild(child);
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
if (expectMigrationError) {
|
||||
await assert.rejects(waitForState(port, child, () => stderr), expectMigrationError);
|
||||
assert.notEqual(child.exitCode, 0);
|
||||
assert.equal(fs.existsSync(markerPath), false);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, 'state.json'), 'utf8')), state);
|
||||
assert.equal(fs.readFileSync(path.join(directory, 'subscription-cache.json'), 'utf8'), cacheContents);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(directory, 'sing-box-config.json'), 'utf8')), config);
|
||||
return { directory, markerPath };
|
||||
}
|
||||
return {
|
||||
directory,
|
||||
markerPath,
|
||||
@@ -257,7 +268,9 @@ test('gateway-direct boot keeps the local proxy and diagnostics but omits every
|
||||
},
|
||||
});
|
||||
const config = JSON.parse(fs.readFileSync(path.join(fixture.directory, 'sing-box-config.json'), 'utf8'));
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(fixture.directory, 'state.json'), 'utf8'));
|
||||
const db = new DatabaseSync(path.join(fixture.directory, 'harbor.sqlite'), { readOnly: true });
|
||||
const stored = JSON.parse(db.prepare("SELECT value FROM documents WHERE key = 'state'").get().value);
|
||||
db.close();
|
||||
|
||||
assert.equal(fixture.state.route.mode, 'gateway-direct');
|
||||
assert.deepEqual(fixture.state.route.activeLocalRules, []);
|
||||
@@ -279,23 +292,15 @@ test('corrupt legacy cache with no canonical subscription fails closed instead o
|
||||
port: 443,
|
||||
protocol: 'vless',
|
||||
};
|
||||
const fixture = await startClientFixture(t, {
|
||||
await startClientFixture(t, {
|
||||
state: { schemaVersion: 4, revision: 2, connectionDesired: 'running' },
|
||||
cacheContents: '{broken',
|
||||
config: generatedConfig(staleServer),
|
||||
expectMigrationError: /Cannot migrate subscription-cache.json/,
|
||||
});
|
||||
|
||||
assert.equal(fixture.state.subscription.status, 'missing');
|
||||
assert.deepEqual(fixture.state.profiles, []);
|
||||
assert.equal(fixture.state.connection.desired, 'stopped');
|
||||
assert.equal(fixture.state.connection.process, 'stopped');
|
||||
assert.equal(fs.existsSync(fixture.markerPath), false);
|
||||
assert.ok(fs.readdirSync(fixture.directory).some((name) => (
|
||||
name.startsWith('subscription-cache.json.corrupt-')
|
||||
)));
|
||||
});
|
||||
|
||||
test('legacy cache owned by another URL is backed up without mixing providers and boots stopped', async (t) => {
|
||||
test('legacy cache owned by another URL aborts migration without mixing providers or changing originals', async (t) => {
|
||||
const stateServer = {
|
||||
id: 'server-a',
|
||||
label: 'State server',
|
||||
@@ -314,7 +319,8 @@ test('legacy cache owned by another URL is backed up without mixing providers an
|
||||
server: 'cache.example',
|
||||
server_port: 8443,
|
||||
};
|
||||
const fixture = await startClientFixture(t, {
|
||||
await startClientFixture(t, {
|
||||
expectMigrationError: /owner mismatch/,
|
||||
state: {
|
||||
schemaVersion: 4,
|
||||
revision: 2,
|
||||
@@ -331,20 +337,6 @@ test('legacy cache owned by another URL is backed up without mixing providers an
|
||||
}),
|
||||
config: generatedConfig(stateServer),
|
||||
});
|
||||
|
||||
assert.equal(fixture.state.profiles.length, 1);
|
||||
assert.equal(fixture.state.profiles[0].subscription.host, 'state.example/…');
|
||||
assert.deepEqual(fixture.state.profiles[0].servers.map(({ id }) => id), [stateServer.id]);
|
||||
assert.equal(JSON.stringify(fixture.state).includes('cache.example'), false);
|
||||
assert.equal(fixture.state.connection.desired, 'stopped');
|
||||
assert.equal(fixture.state.connection.process, 'stopped');
|
||||
assert.equal(fixture.state.selection.appliedServerId, '');
|
||||
assert.equal(fs.existsSync(path.join(fixture.directory, 'subscription-cache.json')), false);
|
||||
assert.ok(fs.readdirSync(fixture.directory).some((name) => (
|
||||
name.startsWith('subscription-cache.json.backup-v1-')
|
||||
)));
|
||||
assert.equal(fs.existsSync(path.join(fixture.directory, 'sing-box-config.json')), false);
|
||||
assert.equal(fs.existsSync(fixture.markerPath), false);
|
||||
});
|
||||
|
||||
test('boot rejects an existing config whose route mode disagrees with the current route', async (t) => {
|
||||
|
||||
@@ -5,6 +5,13 @@ import http from 'node:http';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
|
||||
function readState(directory) {
|
||||
const db = new DatabaseSync(path.join(directory, 'harbor.sqlite'), { readOnly: true });
|
||||
try { return JSON.parse(db.prepare("SELECT value FROM documents WHERE key = 'state'").get().value); }
|
||||
finally { db.close(); }
|
||||
}
|
||||
|
||||
import {
|
||||
assertStateSnapshot,
|
||||
@@ -99,7 +106,7 @@ test('state v1 projects legacy storage through the canonical profile snapshot',
|
||||
);
|
||||
});
|
||||
|
||||
test('startup discards a rejected cached subscription and returns to first-run', async (t) => {
|
||||
test('startup rejects an invalid legacy subscription without changing migration originals', async (t) => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-rejected-cache-'));
|
||||
const port = await freePort();
|
||||
const subscriptionUrl = 'https://provider.example/disabled';
|
||||
@@ -141,16 +148,11 @@ test('startup discards a rejected cached subscription and returns to first-run',
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const state = await waitForState(port, child, () => stderr);
|
||||
assert.equal(state.subscription.status, 'missing');
|
||||
assert.equal(state.hasSubscription, false);
|
||||
assert.deepEqual(state.servers, []);
|
||||
assert.ok(state.route.localRules.some((rule) => (
|
||||
rule.type === 'domain_suffix' && rule.value === 'example.org' && rule.enabled
|
||||
)));
|
||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), false);
|
||||
assert.equal(fs.existsSync(path.join(dir, 'sing-box-config.json')), false);
|
||||
assert.equal(child.exitCode, null);
|
||||
await assert.rejects(waitForState(port, child, () => stderr), /Harbor exited early/);
|
||||
assert.notEqual(child.exitCode, 0);
|
||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), true);
|
||||
assert.equal(fs.readFileSync(path.join(dir, 'sing-box-config.json'), 'utf8'), '{}');
|
||||
assert.equal(JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8')).subscriptionUrl, subscriptionUrl);
|
||||
});
|
||||
|
||||
test('data invariant: canonical profile API mutates one snapshot and preserves migrated profile data', async (t) => {
|
||||
@@ -305,15 +307,15 @@ setInterval(() => {}, 60_000);
|
||||
assert.equal(initial.route.localRulesRevision, 0);
|
||||
assert.equal(initial.route.localRulesPendingRestart, false);
|
||||
assert.equal(JSON.stringify(initial).includes(subscriptionUrl), false);
|
||||
const migratedState = JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'));
|
||||
const migratedState = readState(dir);
|
||||
assert.equal(migratedState.schemaVersion, STATE_SCHEMA_VERSION);
|
||||
assert.equal(migratedState.profiles.length, 1);
|
||||
assert.equal(migratedState.profiles[0].subscriptionUrl, subscriptionUrl);
|
||||
assert.equal(migratedState.profiles[0].desiredServerId, testServerId);
|
||||
assert.equal(migratedState.profiles[0].subscriptionConfig.outbounds[0].tag, testServerId);
|
||||
assert.equal(Object.hasOwn(migratedState, 'subscriptionUrl'), false);
|
||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), false);
|
||||
assert.ok(fs.readdirSync(dir).some((name) => name.startsWith('subscription-cache.json.backup-v1-')));
|
||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), true);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(dir, 'subscription-cache.json'), 'utf8')), { url: subscriptionUrl, config });
|
||||
const stateKeys = Object.keys(initial).sort();
|
||||
assert.deepEqual(stateKeys, [
|
||||
'apiVersion',
|
||||
@@ -373,7 +375,7 @@ setInterval(() => {}, 60_000);
|
||||
|
||||
const primaryProfileId = initial.profiles[0].id;
|
||||
const preservedPrimary = structuredClone(
|
||||
JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8')).profiles[0],
|
||||
readState(dir).profiles[0],
|
||||
);
|
||||
const preservedConfig = fs.readFileSync(path.join(dir, 'sing-box-config.json'), 'utf8');
|
||||
for (const [pathname, expectedCode] of [
|
||||
@@ -394,9 +396,9 @@ setInterval(() => {}, 60_000);
|
||||
},
|
||||
);
|
||||
assert.equal(failedAdd.payload.error.code, expectedCode);
|
||||
const storedAfterFailure = JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'));
|
||||
const storedAfterFailure = readState(dir);
|
||||
assert.deepEqual(storedAfterFailure.profiles, [preservedPrimary]);
|
||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), false);
|
||||
assert.equal(fs.existsSync(path.join(dir, 'subscription-cache.json')), true);
|
||||
assert.equal(
|
||||
fs.readFileSync(path.join(dir, 'sing-box-config.json'), 'utf8'),
|
||||
preservedConfig,
|
||||
@@ -475,7 +477,7 @@ setInterval(() => {}, 60_000);
|
||||
assert.equal(added.state.selection.desiredProfileId, primaryProfileId);
|
||||
assert.equal(added.state.profiles.find(({ id }) => id === workProfileId).desiredServerId, '');
|
||||
assert.equal(
|
||||
JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'))
|
||||
readState(dir)
|
||||
.profiles.find(({ id }) => id === workProfileId).subscriptionConfig.outbounds[0].tag,
|
||||
testServerId,
|
||||
);
|
||||
@@ -517,7 +519,7 @@ setInterval(() => {}, 60_000);
|
||||
);
|
||||
assert.equal(failedRefresh.response.status, 400);
|
||||
assert.equal(failedRefresh.payload.error.code, 'SUBSCRIPTION_INVALID');
|
||||
const storedAfterFailedRefresh = JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'));
|
||||
const storedAfterFailedRefresh = readState(dir);
|
||||
const staleWorkProfile = storedAfterFailedRefresh.profiles.find(({ id }) => id === workProfileId);
|
||||
assert.equal(staleWorkProfile.desiredServerId, testServerId);
|
||||
assert.equal(staleWorkProfile.lastRefreshErrorCode, 'SUBSCRIPTION_INVALID');
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { createTrafficHistoryRoute } from '../../dist/server/http/routes/trafficHistoryRoute.js';
|
||||
import { emptyTrafficHistory, parseTrafficHistoryQuery } from '../../dist/shared/trafficHistory.js';
|
||||
|
||||
function response() {
|
||||
return { writeHead(status) { this.status = status; }, end(body) { this.payload = JSON.parse(body); } };
|
||||
}
|
||||
|
||||
test('history API reads only its local source, enriches labels and returns explicit unavailable coverage', async () => {
|
||||
const route = createTrafficHistoryRoute({
|
||||
readHistory: async (query) => ({ ...emptyTrafficHistory(query, 'live'), origins: [{ id: 'dev-a', label: 'IP' }] }),
|
||||
deviceInventory: { snapshot: () => ({ devices: [{ id: 'dev-a', alias: 'Ноутбук' }] }) },
|
||||
});
|
||||
const res = response();
|
||||
assert.equal(await route.handle({ method: 'GET', url: '/api/traffic/history?range=90d&search=yandex' }, res), true);
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.payload.query.search, 'yandex');
|
||||
assert.equal(res.payload.period.retentionDays, 90);
|
||||
assert.equal(res.payload.origins[0].label, 'Ноутбук');
|
||||
const failed = createTrafficHistoryRoute({ readHistory: async () => { throw Error('disk'); } });
|
||||
await failed.handle({ method: 'GET', url: '/api/traffic/history' }, res);
|
||||
assert.equal(res.payload.storage.status, 'error');
|
||||
assert.equal(res.payload.coverage.partial, true);
|
||||
await createTrafficHistoryRoute({ readHistory: null }).handle({ method: 'GET', url: '/api/traffic/history' }, res);
|
||||
assert.equal(res.payload.source, 'disabled');
|
||||
});
|
||||
|
||||
test('history API rejects invalid queries and mutation methods before reading storage', async () => {
|
||||
let reads = 0;
|
||||
const route = createTrafficHistoryRoute({ readHistory: async (query) => { reads++; return emptyTrafficHistory(query); } });
|
||||
for (const params of ['range=forever', 'level=ip%3BDROP', 'route=no', 'offset=-1', 'until=Infinity', 'search=%00', `search=${'x'.repeat(201)}`]) {
|
||||
await assert.rejects(route.handle({ method: 'GET', url: `/api/traffic/history?${params}` }, response()),
|
||||
(error) => error.code === 'REQUEST_INVALID');
|
||||
}
|
||||
await assert.rejects(route.handle({ method: 'DELETE', url: '/api/traffic/history' }, response()),
|
||||
(error) => error.code === 'ENDPOINT_NOT_FOUND');
|
||||
assert.equal(await route.handle({ method: 'GET', url: '/api/state' }, response()), false);
|
||||
assert.equal(reads, 0);
|
||||
assert.equal(parseTrafficHistoryQuery(new URLSearchParams('range=90d')).range, '90d');
|
||||
});
|
||||
@@ -0,0 +1,299 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import test from 'node:test';
|
||||
import { openTrafficHistoryStore } from '../../dist/server/services/trafficHistoryStore.js';
|
||||
import { createTrafficHistoryService } from '../../dist/server/services/trafficHistoryService.js';
|
||||
import { openHarborStorage } from '../../dist/server/services/harborStorage.js';
|
||||
import { createDomainTrafficService } from '../../dist/server/services/domainTrafficService.js';
|
||||
import { assertTrafficHistorySnapshot, parseTrafficHistoryQuery } from '../../dist/shared/trafficHistory.js';
|
||||
|
||||
const DAY = 86_400_000;
|
||||
const base = Date.parse('2026-06-01T10:00:00.000Z');
|
||||
const query = (fields = {}) => ({ ...parseTrafficHistoryQuery(new URLSearchParams('range=90d')), ...fields });
|
||||
function connection(id, up, down, fields = {}) {
|
||||
return {
|
||||
id, startedAt: new Date(base + 1_000).toISOString(), closedAt: null,
|
||||
inbound: { tag: 'tproxy-in', type: 'tproxy' }, network: 'tcp', protocol: 'tls',
|
||||
source: { ip: '192.0.2.10', port: 50_000 },
|
||||
destination: { domain: 'www.yandex.ru', ip: '203.0.113.10', port: 443, provenance: 'sing-box' },
|
||||
origin: { kind: 'device', id: 'dev_0123456789abcdef', label: 'Laptop', provenance: 'source-ip' },
|
||||
route: { kind: 'vpn', scope: 'local-sing-box', outbound: 'vpn-one', outboundType: 'vless', chain: [], rule: null },
|
||||
traffic: { uploadBytes: String(up), downloadBytes: String(down), uploadBytesPerSecond: '0', downloadBytesPerSecond: '0' },
|
||||
...fields,
|
||||
};
|
||||
}
|
||||
const batch = (at, connections = [], reset = false, epoch = 'sing-box-1') => ({ epoch,
|
||||
observedAt: new Date(at).toISOString(), connections, reset, closedIds: connections.filter((c) => c.closedAt).map((c) => c.id) });
|
||||
function fixture(t) {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-traffic-sql-'));
|
||||
const closers = new Set();
|
||||
t.after(async () => {
|
||||
for (const close of closers) await close();
|
||||
fs.rmSync(directory, { recursive: true, force: true });
|
||||
});
|
||||
return { directory, file: path.join(directory, 'traffic.sqlite'), register(store) {
|
||||
const original = store.close;
|
||||
store.close = () => { closers.delete(store.close); return original(); };
|
||||
closers.add(store.close);
|
||||
return store;
|
||||
} };
|
||||
}
|
||||
|
||||
test('SQL checkpoints survive reopen, reset/replayed CLOSED and same-UUID new lifecycles without double count', (t) => {
|
||||
const f = fixture(t);
|
||||
let clock = base;
|
||||
let store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live');
|
||||
clock += 1_000;
|
||||
store.ingest([batch(clock, [connection('a', 10, 20)])], 'live');
|
||||
clock += 1_000;
|
||||
store.ingest([batch(clock, [connection('a', 15, 27)])], 'live');
|
||||
store.close();
|
||||
store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [connection('a', 15, 27)], true)], 'live');
|
||||
clock += 1_000;
|
||||
const closed = connection('a', 18, 30, { closedAt: new Date(clock).toISOString() });
|
||||
store.ingest([batch(clock, [closed])], 'live');
|
||||
store.close();
|
||||
store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
clock += 1_000;
|
||||
store.ingest([batch(clock, [closed], true)], 'live');
|
||||
clock += 60_000;
|
||||
assert.deepEqual(store.query(query()).totals, { uploadBytes: '18', downloadBytes: '30' });
|
||||
store.ingest([batch(clock, [connection('a', 7, 9, { startedAt: new Date(clock).toISOString() })])], 'live');
|
||||
clock += 60_000;
|
||||
assert.deepEqual(store.query(query()).totals, { uploadBytes: '25', downloadBytes: '39' });
|
||||
const result = assertTrafficHistorySnapshot(store.query(query()));
|
||||
assert.equal(result.rows[0].label, 'Яндекс');
|
||||
});
|
||||
|
||||
test('full hostname, PSL registered domain, separate IP/device/routes and IP-only destinations remain queryable', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||
store.ingest([batch(clock, [
|
||||
connection('a', 10, 20),
|
||||
connection('b', 3, 4, { destination: { domain: 'mail.yandex.com', ip: '203.0.113.20' } }),
|
||||
connection('c', 5, 6, { destination: { domain: 'api.example.co.uk', ip: '203.0.113.30' } }),
|
||||
connection('d', 7, 8, { destination: { domain: null, ip: '2001:db8::1' }, origin: { kind: 'unknown', id: null, label: 'Unknown' } }),
|
||||
])], 'live');
|
||||
clock += 60_000;
|
||||
const roots = store.query(query());
|
||||
assert.deepEqual(new Set(roots.rows.map((row) => row.key)), new Set(['Яндекс', 'example.co.uk', '']));
|
||||
const domains = store.query(query({ level: 'domain', service: 'Яндекс' }));
|
||||
assert.deepEqual(new Set(domains.rows.map((row) => row.key)), new Set(['yandex.ru', 'yandex.com']));
|
||||
assert.equal(store.query(query({ level: 'hostname', service: 'Яндекс', domain: 'yandex.ru' })).rows[0].key, 'www.yandex.ru');
|
||||
const ip = store.query(query({ level: 'ip', service: 'Яндекс', domain: 'yandex.ru', hostname: 'www.yandex.ru' }));
|
||||
assert.equal(ip.rows[0].key, '203.0.113.10');
|
||||
assert.equal(store.query(query({ level: 'ip', service: '', domain: '', hostname: '' })).rows[0].key, '2001:db8::1');
|
||||
assert.equal(store.query(query({ originId: 'unknown:192.0.2.10' })).totals.downloadBytes, '8');
|
||||
assert.equal(store.query(query({ search: "x' OR 1=1 --" })).rows.length, 0);
|
||||
assert.equal(store.query(query({ search: 'яндекс' })).totals.downloadBytes, '24');
|
||||
});
|
||||
|
||||
test('initial snapshot is a baseline and downtime is explicitly partial', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [connection('old', 1_000, 2_000, { startedAt: new Date(base - DAY).toISOString() })], true)], 'live');
|
||||
clock += 1_000;
|
||||
assert.equal(store.query(query()).totals.uploadBytes, '0');
|
||||
store.ingest([batch(clock, [connection('old', 1_010, 2_020, { startedAt: new Date(base - DAY).toISOString() })])], 'live');
|
||||
clock += 60_000;
|
||||
store.ingest([batch(clock, [connection('old', 1_040, 2_050, { startedAt: new Date(base - DAY).toISOString() })], true)], 'live');
|
||||
clock += 60_000;
|
||||
assert.deepEqual(store.query(query()).totals, { uploadBytes: '40', downloadBytes: '50' });
|
||||
assert.equal(store.query(query()).coverage.partial, true);
|
||||
});
|
||||
|
||||
test('counter regression retains high watermarks and reconnect counts new post-collection lifecycles', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live');
|
||||
for (const total of [100, 90, 100]) {
|
||||
clock += 1_000;
|
||||
store.ingest([batch(clock, [connection('a', total, total)], true)], 'live');
|
||||
}
|
||||
clock += 60_000;
|
||||
const result = store.query(query());
|
||||
assert.deepEqual(result.totals, { uploadBytes: '100', downloadBytes: '100' });
|
||||
assert.equal(result.coverage.partial, true);
|
||||
});
|
||||
|
||||
test('fast collector reopen marks missing coverage even if the next source state is live', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
let store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live');
|
||||
store.close(); clock += 1_000;
|
||||
store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [connection('a', 10, 20)], true)], 'live');
|
||||
clock += 60_000;
|
||||
assert.equal(store.query(query()).coverage.gapCount, 1);
|
||||
});
|
||||
|
||||
test('frozen history cutoff excludes its incomplete terminal bucket before and after hourly rollup', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||
store.ingest([batch(clock, [connection('a', 100, 200)])], 'live');
|
||||
clock += 60_000;
|
||||
store.ingest([batch(clock, [connection('a', 110, 220)])], 'live');
|
||||
const frozen = store.query(query());
|
||||
assert.equal(frozen.query.until, base + 60_000);
|
||||
assert.equal(frozen.totals.uploadBytes, '100');
|
||||
clock += 1_000;
|
||||
store.ingest([batch(clock, [connection('a', 120, 240)])], 'live');
|
||||
assert.deepEqual(store.query(frozen.query).totals, frozen.totals);
|
||||
clock = base + 8 * DAY;
|
||||
const hourly = store.query(query({ until: base + 3_600_000 + 30_000 }));
|
||||
assert.equal(hourly.query.until, base + 3_600_000);
|
||||
assert.equal(hourly.totals.uploadBytes, '120');
|
||||
const expired = store.query(query({ until: base - 100 * DAY }));
|
||||
assert.equal(expired.rows.length, 0);
|
||||
assert.equal(expired.period.from, expired.period.to);
|
||||
});
|
||||
|
||||
test('minute to hour rollup preserves exact bytes and identities; 90-day cleanup leaves settings and metrics untouched', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
const settings = f.register(openHarborStorage(f.directory));
|
||||
settings.state.update((state) => ({ ...state, revision: 42 }));
|
||||
const metrics = createDomainTrafficService({ observe: () => ({ connections: [] }), devices: () => [] });
|
||||
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||
const data = batch(clock, [connection('a', '9007199254740993', '500')]);
|
||||
store.ingest([data], 'live'); metrics.ingestNative(data);
|
||||
const metricsBefore = JSON.stringify(metrics.snapshot());
|
||||
clock += 60_000;
|
||||
store.ingest([batch(clock, [connection('a', '9007199254741000', '550')])], 'live');
|
||||
clock = base + 8 * DAY;
|
||||
store.maintain(); store.maintain();
|
||||
assert.deepEqual(store.query(query()).totals, { uploadBytes: '9007199254741000', downloadBytes: '550' });
|
||||
const inspect = new DatabaseSync(f.file);
|
||||
assert.deepEqual(inspect.prepare('SELECT DISTINCT resolution FROM buckets').all().map((r) => r.resolution), [3_600_000]);
|
||||
inspect.close();
|
||||
clock = base + 91 * DAY;
|
||||
store.maintain();
|
||||
assert.equal(store.query(query()).rows.length, 0);
|
||||
assert.equal(settings.state.read().revision, 42);
|
||||
assert.equal(JSON.stringify(metrics.snapshot()), metricsBefore);
|
||||
});
|
||||
|
||||
test('a failed bucket transaction does not advance checkpoints, and retry counts exactly once', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||
const blocker = new DatabaseSync(f.file);
|
||||
blocker.exec("CREATE TRIGGER reject_bucket BEFORE INSERT ON buckets BEGIN SELECT RAISE(ABORT, 'simulated disk failure'); END");
|
||||
const data = batch(clock, [connection('a', 10, 20)]);
|
||||
assert.throws(() => store.ingest([data], 'live'), /simulated/);
|
||||
assert.equal(blocker.prepare('SELECT COUNT(*) AS n FROM checkpoints').get().n, 0);
|
||||
blocker.exec('DROP TRIGGER reject_bucket'); blocker.close();
|
||||
store.ingest([data], 'live'); store.ingest([data], 'live'); clock += 60_000;
|
||||
assert.deepEqual(store.query(query()).totals, { uploadBytes: '10', downloadBytes: '20' });
|
||||
});
|
||||
|
||||
test('idle active checkpoints survive retention; a new runtime epoch removes old checkpoints but not buckets', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||
store.ingest([batch(clock, [connection('idle', 100, 200)])], 'live');
|
||||
clock += 91 * DAY;
|
||||
store.maintain();
|
||||
store.ingest([batch(clock, [connection('idle', 110, 220)])], 'live');
|
||||
clock += 60_000;
|
||||
assert.equal(store.query(query()).totals.uploadBytes, '10');
|
||||
store.ingest([batch(clock, [], true, 'sing-box-2')], 'live');
|
||||
const db = new DatabaseSync(f.file);
|
||||
assert.equal(db.prepare('SELECT COUNT(*) AS n FROM checkpoints').get().n, 0);
|
||||
db.close();
|
||||
assert.equal(store.query(query()).totals.uploadBytes, '10');
|
||||
});
|
||||
|
||||
test('reset disappearance and terminal identities retire checkpoints without forgetting still-active baselines', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live'); clock += 1_000;
|
||||
store.ingest([batch(clock, [connection('gone', 10, 20), connection('idle', 100, 200), connection('terminal', 7, 8)])], 'live');
|
||||
clock += 1_000;
|
||||
store.ingest([{ ...batch(clock), closedIds: ['terminal'] }], 'live');
|
||||
clock += 1_000;
|
||||
store.ingest([batch(clock, [connection('idle', 100, 200)], true)], 'live');
|
||||
clock += 91 * DAY;
|
||||
store.maintain();
|
||||
const db = new DatabaseSync(f.file);
|
||||
assert.deepEqual(db.prepare("SELECT json_extract(identity, '$[1]') AS id, closed FROM checkpoints").all()
|
||||
.map((row) => ({ ...row })), [{ id: 'idle', closed: 0 }]);
|
||||
db.close();
|
||||
store.ingest([batch(clock, [connection('idle', 110, 220)])], 'live');
|
||||
clock += 60_000;
|
||||
assert.equal(store.query(query()).totals.uploadBytes, '10');
|
||||
});
|
||||
|
||||
test('worker ingestion is independent of Prometheus and errors remain isolated; many closed events are not UI-capped', async (t) => {
|
||||
const f = fixture(t);
|
||||
const service = f.register(createTrafficHistoryService({ filePath: f.file, source: () => 'live' }));
|
||||
const at = Date.now() - 120_000;
|
||||
service.enqueue(batch(at, [], true));
|
||||
service.enqueue(batch(at + 1, Array.from({ length: 2_049 }, (_, index) => connection(`closed-${index}`, 1, 2, {
|
||||
startedAt: new Date(at + 1).toISOString(), closedAt: new Date(at + 1).toISOString(),
|
||||
}))));
|
||||
let ticked = false;
|
||||
setImmediate(() => { ticked = true; });
|
||||
const result = await service.query(query());
|
||||
assert.equal(ticked, true);
|
||||
assert.equal(result.storage.status, 'ready');
|
||||
assert.deepEqual(result.totals, { uploadBytes: '2049', downloadBytes: '4098' });
|
||||
const invalid = f.register(createTrafficHistoryService({ filePath: f.directory, source: () => 'live' }));
|
||||
assert.doesNotThrow(() => invalid.enqueue(batch(at, [])));
|
||||
assert.equal((await invalid.query(query())).storage.status, 'error');
|
||||
});
|
||||
|
||||
test('bounded 90-day history performance sample reports write/query size without claiming a hardware guarantee', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live');
|
||||
const start = performance.now();
|
||||
for (let day = 0; day < 90; day++) {
|
||||
clock = base + day * DAY + 1_000;
|
||||
store.ingest([batch(clock, Array.from({ length: 100 }, (_, index) => connection(`${day}-${index}`, 100, 1_000, {
|
||||
startedAt: new Date(clock).toISOString(), destination: { domain: `host${index}.example.org`, ip: `203.0.113.${index + 1}` },
|
||||
})))], 'live');
|
||||
}
|
||||
const writeMs = performance.now() - start;
|
||||
clock += 60_000;
|
||||
const queryStart = performance.now();
|
||||
const result = store.query(query());
|
||||
const queryMs = performance.now() - queryStart;
|
||||
assert.equal(result.totals.downloadBytes, '9000000');
|
||||
const sizes = Object.fromEntries(['', '-wal'].map((suffix) => [suffix || 'db', fs.statSync(f.file + suffix).size]));
|
||||
t.diagnostic(JSON.stringify({ samples: 9_000, simulatedDays: 90, writeMs, queryMs, sizes }));
|
||||
});
|
||||
|
||||
test('high-churn sample measures closed-lifecycle storage and epoch reclamation', (t) => {
|
||||
const f = fixture(t); let clock = base;
|
||||
const store = f.register(openTrafficHistoryStore(f.file, () => clock));
|
||||
store.ingest([batch(clock, [], true)], 'live');
|
||||
const start = performance.now();
|
||||
for (let page = 0; page < 100; page++) {
|
||||
clock += 1_000;
|
||||
store.ingest([batch(clock, Array.from({ length: 1_000 }, (_, index) => connection(`${page}-${index}`, 100, 1_000, {
|
||||
startedAt: new Date(clock).toISOString(), closedAt: new Date(clock).toISOString(),
|
||||
})))], 'live');
|
||||
}
|
||||
const writeMs = performance.now() - start;
|
||||
clock += 60_000;
|
||||
const startQuery = performance.now();
|
||||
assert.equal(store.query(query()).totals.downloadBytes, '100000000');
|
||||
const queryMs = performance.now() - startQuery;
|
||||
const db = new DatabaseSync(f.file);
|
||||
assert.equal(db.prepare('SELECT COUNT(*) AS n FROM checkpoints').get().n, 100_000);
|
||||
const bytes = db.prepare('PRAGMA page_count').get().page_count * db.prepare('PRAGMA page_size').get().page_size;
|
||||
store.ingest([batch(clock, [], true, 'sing-box-next')], 'live');
|
||||
assert.equal(db.prepare('SELECT COUNT(*) AS n FROM checkpoints').get().n, 0);
|
||||
const reusableBytes = db.prepare('PRAGMA freelist_count').get().freelist_count * db.prepare('PRAGMA page_size').get().page_size;
|
||||
db.close();
|
||||
t.diagnostic(JSON.stringify({ closedLifecycles: 100_000, writeMs, queryMs, bytes, reusableBytes }));
|
||||
});
|
||||
Reference in New Issue
Block a user