Add domain traffic metrics and Grafana dashboard

test/server/dataplane-client.test.js

@@ -24,6 +24,8 @@ test('control uses the dataplane socket protocol', async () => {
   assert.equal(devices.running, true);
   const traffic = await client.observeTraffic();
   assert.equal(traffic.running, true);
+  const domainTraffic = await client.observeDomainTraffic();
+  assert.equal(domainTraffic.running, true);
   await client.observeDevicePolicy();
   await client.applyDevicePolicies([{ id: 'dev_0011223344556677' }]);
   await client.runConnectivityDiagnostics(
@@ -38,18 +40,19 @@ test('control uses the dataplane socket protocol', async () => {
     'POST /apply /run/dataplane.sock',
     'GET /devices /run/dataplane.sock',
     'GET /device-traffic /run/dataplane.sock',
+    'GET /domain-traffic /run/dataplane.sock',
     'GET /device-policy /run/dataplane.sock',
     'PUT /device-policy /run/dataplane.sock',
     'POST /diagnostics/connectivity /run/dataplane.sock',
     'POST /restart /run/dataplane.sock',
     'POST /stop /run/dataplane.sock',
   ]);
-  assert.deepEqual(requests[5].body, { devices: [{ id: 'dev_0011223344556677' }] });
-  assert.deepEqual(requests[6].body, {
+  assert.deepEqual(requests[6].body, { devices: [{ id: 'dev_0011223344556677' }] });
+  assert.deepEqual(requests[7].body, {
     services: [{ id: 'custom-test', url: 'https://example.com' }],
     target: 'site:custom-test',
   });
-  assert.equal(requests[6].timeoutMs, 25_000);
+  assert.equal(requests[7].timeoutMs, 25_000);
 });
 
 test('connectivity diagnostics expose a retryable domain error', async () => {

test/server/device-inventory.test.js

@@ -848,3 +848,19 @@ test('malformed proxy totals are backed up and an expired recovery marker is cle
   assert.deepEqual(store.read().traffic.proxy.rebaselineMacs, []);
   assert.equal(snapshot.source.traffic.proxy.error, null);
 });
+
+test('an old dataplane without domain traffic keeps inventory refresh and existing metrics available', async (t) => {
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'harbor-domain-compat-'));
+  t.after(() => fs.rmSync(directory, { recursive: true, force: true }));
+  const store = createJsonStore({ filePath: path.join(directory, 'devices.json'), defaultValue: {} });
+  const service = createDeviceInventoryService({
+    store,
+    observe: () => ({ observedAt: '2026-08-08T10:00:00.000Z', observations: [], error: null }),
+    observeDomainTraffic: () => { throw new Error('Dataplane HTTP 404'); },
+  });
+
+  const snapshot = await service.refresh();
+  assert.equal(snapshot.devices.length, 0);
+  assert.equal(service.metricsSnapshot().domainTraffic.source.error, 'Dataplane HTTP 404');
+  assert.equal(service.metricsSnapshot().traffic.totalBytes, '0');
+});

test/server/domain-traffic.test.js

@@ -0,0 +1,103 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import {
+  classifyDomain,
+  createDomainTrafficService,
+} from '../../src/server/services/domainTrafficService.js';
+import { deviceId } from '../../src/server/services/deviceInventoryService.js';
+
+const mac = '00:11:22:33:44:55';
+const id = deviceId(mac);
+const device = { ip: '192.168.50.7', mac };
+const connection = (connectionId, type, host, upload, download, sourceIP = device.ip) => ({
+  id: connectionId,
+  metadata: { type, host, sourceIP },
+  upload,
+  download,
+});
+
+test('domain traffic accumulates connection deltas by device, service and source', async () => {
+  let observedAt = new Date('2026-08-08T10:00:00.000Z');
+  let response = { connections: [
+    connection('youtube', 'tproxy/tproxy-in', 'r1.googlevideo.com', 10, 100),
+    connection('chatgpt', 'mixed/mixed-in', 'www.chatgpt.com.', 20, 200),
+    connection('diagnostics', 'mixed/diagnostics-vpn-in', 'example.com', 30, 300),
+    connection('unknown-device', 'tproxy/tproxy-in', 'example.net', 40, 400, '192.168.50.99'),
+  ] };
+  const service = createDomainTrafficService({
+    observe: async () => response,
+    devices: () => [device],
+    now: () => observedAt,
+  });
+
+  await service.refresh();
+  response = { connections: [
+    connection('youtube', 'tproxy/tproxy-in', 'r1.googlevideo.com', 15, 130),
+    connection('chatgpt', 'mixed/mixed-in', 'www.chatgpt.com.', 22, 260),
+  ] };
+  observedAt = new Date('2026-08-08T10:00:02.000Z');
+  await service.refresh();
+  await service.refresh();
+
+  assert.deepEqual(service.snapshot().series, [
+    {
+      deviceId: id,
+      domain: 'chatgpt.com',
+      service: 'OpenAI / ChatGPT',
+      source: 'proxy',
+      uploadBytes: '22',
+      downloadBytes: '260',
+    },
+    {
+      deviceId: id,
+      domain: 'googlevideo.com',
+      service: 'YouTube',
+      source: 'gateway',
+      uploadBytes: '15',
+      downloadBytes: '130',
+    },
+  ]);
+  assert.equal(service.snapshot().observedAt, observedAt.toISOString());
+  assert.equal(service.snapshot().source.error, null);
+
+  response = { connections: [] };
+  await service.refresh();
+  assert.equal(service.snapshot().series[1].downloadBytes, '130');
+});
+
+test('domain traffic is bounded and keeps the last good snapshot on source failure', async () => {
+  let fail = false;
+  const service = createDomainTrafficService({
+    observe: async () => {
+      if (fail) throw new Error('Clash API unavailable');
+      return { connections: [
+        connection('first', 'tproxy/tproxy-in', 'one.example', 1, 10),
+        connection('second', 'tproxy/tproxy-in', 'two.example', 2, 20),
+      ] };
+    },
+    devices: () => [device],
+    maxSeries: 3,
+  });
+
+  await service.refresh();
+  await service.refresh();
+  assert.deepEqual(
+    Object.fromEntries(service.snapshot().series.map(({ domain, downloadBytes }) => [domain, downloadBytes])),
+    { 'one.example': '10', _other: '20' },
+  );
+  assert.equal(service.snapshot().overflowConnections, '1');
+  assert.equal(service.snapshot().series.find(({ domain }) => domain === '_other').deviceId, '_other');
+  assert.ok(service.snapshot().series.length <= 3);
+
+  fail = true;
+  await assert.rejects(service.refresh(), /Clash API unavailable/);
+  assert.equal(service.snapshot().series.find(({ domain }) => domain === 'one.example').downloadBytes, '10');
+  assert.equal(service.snapshot().source.error, 'Clash API unavailable');
+});
+
+test('domain classification normalizes known services and rejects IP or malformed labels', () => {
+  assert.deepEqual(classifyDomain('WWW.YouTube.com.'), { domain: 'youtube.com', service: 'YouTube' });
+  assert.deepEqual(classifyDomain('api.example.org'), { domain: 'api.example.org', service: 'api.example.org' });
+  assert.equal(classifyDomain('192.0.2.1'), null);
+  assert.equal(classifyDomain('broken_label.example'), null);
+});

test/server/prometheus-metrics.test.js

@@ -26,6 +26,18 @@ const snapshot = {
     proxyDownloadBytes: '0',
     proxyTrafficObservedAt: null,
   }],
+  domainTraffic: {
+    observedAt,
+    overflowConnections: '2',
+    series: [{
+      deviceId: 'dev_0011223344556677',
+      domain: 'chatgpt.com',
+      service: 'OpenAI / ChatGPT',
+      source: 'proxy',
+      uploadBytes: '12',
+      downloadBytes: '345',
+    }],
+  },
 };
 
 test('Prometheus exposition keeps exact counters, stable identity and escaped names', () => {
@@ -39,6 +51,10 @@ test('Prometheus exposition keeps exact counters, stable identity and escaped na
   assert.doesNotMatch(output, /harbor_device_traffic_bytes_total\{[^\n]*source="proxy"/);
   assert.doesNotMatch(output, /00:11:22:33:44:55/);
   assert.match(output, /harbor_device_traffic_last_observed_timestamp_seconds\{device_id="dev_0011223344556677",source="gateway"\} 1786183200/);
+  assert.match(output, /harbor_device_domain_traffic_bytes_total\{device_id="dev_0011223344556677",domain="chatgpt\.com",service="OpenAI \/ ChatGPT",source="proxy",direction="download"\} 345/);
+  assert.match(output, /harbor_domain_traffic_last_observed_timestamp_seconds 1786183200/);
+  assert.match(output, /harbor_domain_traffic_overflow_connections_total 2/);
+  assert.doesNotMatch(output, /harbor_device_domain_traffic_bytes_total\{[^\n]*name=/);
   assert.equal(output.endsWith('\n'), true);
 });
 

test/server/singbox-gateway-mode.test.js

@@ -46,5 +46,6 @@ test('gateway routes .ru domains directly and other traffic through the selected
     { inbound: ['tproxy-in'], outbound: 'test-vpn' },
     { inbound: ['mixed-in'], outbound: 'test-vpn' },
   ]);
+  assert.deepEqual(config.experimental.clash_api, { external_controller: '127.0.0.1:19090' });
   assert.equal(config.route.final, 'test-vpn');
 });